pytorch_test.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Testing framework for comparison of pytorch and candle results\n",
    "\n",
    "<a target=\"_blank\" href=\"https://colab.research.google.com/github/KGrewal1/optimisers/blob/master/pytorch_test.ipynb\">\n",
    "  <img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.0\n",
      "12.1\n",
      "NVIDIA GeForce RTX 3070 Ti\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch import optim\n",
    "print(torch.__version__)\n",
    "print(torch.version.cuda)\n",
    "print(torch.cuda.get_device_name(0)) # this should be commented out to run on colab"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SGD tests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## nesterov_sgd_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[ 1.0750, -9.9042]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-1.8961], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.SGD(m.parameters(), lr=0.004, momentum=0.1, nesterov=True)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## nesterov_decay_sgd_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[  0.9921, -10.3803]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-1.9331], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.SGD(m.parameters(), lr=0.004, momentum=0.1, nesterov=True, weight_decay = 0.1)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## momentum_sgd_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[2.8870, 0.8589]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.6341], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.SGD(m.parameters(), lr=0.004, momentum=0.1, nesterov=False, weight_decay = 0.0)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## momentum_sgd_decay_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[2.8751, 0.8514]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.5626], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.SGD(m.parameters(), lr=0.004, momentum=0.1, nesterov=False, weight_decay = 0.4)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## momentum_sgd_dampened_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[2.8746, 0.8434]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.4838], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.SGD(m.parameters(), lr=0.004, momentum=0.1, nesterov=False, weight_decay = 0.0, dampening=0.2)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## sgd_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[2.8809, 0.8513]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.5606], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.SGD(m.parameters(), lr=0.004, weight_decay = 0.0)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## sgd_decay_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[2.8700, 0.8450]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([-0.5003], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.SGD(m.parameters(), lr=0.004, weight_decay = 0.4)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AdaGrad tests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## adagrad_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[0.2424, 0.2341]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([0.2379], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.Adagrad(m.parameters(), lr=0.004, weight_decay=0.00)\n",
    "for _step in range(1000):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## adagrad_lr_decay_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[0.0231, 0.0230]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([0.0230], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.Adagrad(m.parameters(), lr=0.004, lr_decay=0.2)\n",
    "for _step in range(1000):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## adagrad_weight_decay_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[0.2424, 0.2341]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([0.2378], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.Adagrad(m.parameters(), lr=0.004, weight_decay=0.2)\n",
    "for _step in range(1000):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AdaDelta Tests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## adadelta_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[0.0016, 0.0016]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([0.0016], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.Adadelta(m.parameters(), lr=0.004)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## adadelta_weight_decay_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[0.0016, 0.0016]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([0.0016], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.Adadelta(m.parameters(), lr=0.004, weight_decay = 0.8)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AdaMax Tests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## adamax_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[0.3895, 0.3450]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([0.3643], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.Adamax(m.parameters(), lr=0.004)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## adamax_weight_decay_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[0.3894, 0.3450]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([0.3639], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.Adamax(m.parameters(), lr=0.004, weight_decay = 0.6)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NAdam Tests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## nadam_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[0.1897, 0.1837]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([0.1864], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.NAdam(m.parameters())\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## nadam_weight_decay_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[0.1897, 0.1837]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([0.1863], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.NAdam(m.parameters(), weight_decay = 0.6)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## nadam_decoupled_weight_decay_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[0.1792, 0.1737]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([0.1762], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.NAdam(m.parameters(), weight_decay = 0.6, decoupled_weight_decay=True)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RAdam Tests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## radam_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[2.2128, 1.2819]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([0.2923], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.RAdam(m.parameters())\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## radam_weight_decay_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[2.2117, 1.2812]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([0.2921], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.RAdam(m.parameters(), weight_decay = 0.4)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# RMSprop Tests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## rmsprop_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[1.6650, 0.7867]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([1.3012], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.RMSprop(m.parameters())\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## rmsprop_weight_decay_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[1.6643, 0.7867]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([1.2926], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.RMSprop(m.parameters(), weight_decay = 0.4)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## rmsprop_centered_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[1.8892, 0.7617]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([1.3688], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.RMSprop(m.parameters(), centered = True)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## rmsprop_centered_decay_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[1.8883, 0.7621]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([1.3558], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.RMSprop(m.parameters(), centered = True, weight_decay = 0.4)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## rmsprop_momentum_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[2.3042, 0.6835]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([1.5441], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.RMSprop(m.parameters(),  momentum = 0.4)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## rmsprop_momentum_decay_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[2.3028, 0.6858]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([1.5149], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.RMSprop(m.parameters(),  momentum = 0.4, weight_decay = 0.4)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## rmsprop_centered_momentum_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[2.4486, 0.6715]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([1.5045], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.RMSprop(m.parameters(), centered = True, momentum = 0.4)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## rmsprop_centered_momentum_decay_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[2.4468, 0.6744]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([1.4695], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.RMSprop(m.parameters(), centered = True, momentum = 0.4, weight_decay = 0.4)\n",
    "for _step in range(100):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Adam Tests"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## adam_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[0.9000, 0.6967]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([0.7996], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch import optim\n",
    "\n",
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.Adam(m.parameters())\n",
    "for _step in range(1000):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## adam_weight_decay_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[0.8997, 0.6964]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([0.7975], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch import optim\n",
    "\n",
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.Adam(m.parameters(), weight_decay = 0.6)\n",
    "for _step in range(1000):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## adamw_weight_decay_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[0.6901, 0.5677]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([0.6287], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch import optim\n",
    "\n",
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.AdamW(m.parameters(), weight_decay = 0.6)\n",
    "for _step in range(1000):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## adam_amsgrad_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[0.9001, 0.6904]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([0.7978], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch import optim\n",
    "\n",
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.Adam(m.parameters(), amsgrad=True)\n",
    "for _step in range(1000):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## adam_amsgrad_decay_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[0.8998, 0.6901]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([0.7955], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch import optim\n",
    "\n",
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.Adam(m.parameters(), amsgrad=True, weight_decay=0.6)\n",
    "for _step in range(1000):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## adamw_amsgrad_decay_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parameter containing:\n",
      "tensor([[0.6901, 0.5648]], requires_grad=True)\n",
      "Parameter containing:\n",
      "tensor([0.6287], requires_grad=True)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch import optim\n",
    "\n",
    "w_gen = torch.tensor([[3., 1.]])\n",
    "b_gen = torch.tensor([-2.])\n",
    "\n",
    "sample_xs = torch.tensor([[2., 1.], [7., 4.], [-4., 12.], [5., 8.]])\n",
    "sample_ys = sample_xs.matmul(w_gen.t()) + b_gen\n",
    "\n",
    "m = torch.nn.Linear(2, 1)\n",
    "with torch.no_grad():\n",
    "    m.weight.zero_()\n",
    "    m.bias.zero_()\n",
    "optimiser = optim.AdamW(m.parameters(), amsgrad=True, weight_decay=0.6)\n",
    "for _step in range(1000):\n",
    "    optimiser.zero_grad()\n",
    "    ys = m(sample_xs)\n",
    "    loss = ((ys - sample_ys)**2).sum()\n",
    "    loss.backward()\n",
    "    optimiser.step()\n",
    "print(m.weight)\n",
    "print(m.bias)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}