075_pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pytorch Introduction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tensor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 2.],\n",
       "        [3., 4.]])"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.empty(1,2) # uninitalized\n",
    "torch.rand(1,2) # ~ unif(0,1)\n",
    "torch.randn(1,2) # ~ Normal(0,1)\n",
    "torch.zeros(1,2)\n",
    "torch.ones(1,2)\n",
    "torch.eye(3) # identity matrix\n",
    "torch.Tensor([[1,2],[3,4]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[3., 3.]])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.ones(1,2) * 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 1.]])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.zeros(1,2) + torch.ones(1,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0.],\n",
       "        [0., 1., 0.],\n",
       "        [0., 0., 1.]])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.ones(3) * torch.eye(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 tensor([0.4963], requires_grad=True)\n",
      "100 tensor([-0.3657], requires_grad=True) tensor([0.4108], grad_fn=<AddBackward0>)\n",
      "200 tensor([-0.8187], requires_grad=True) tensor([0.0339], grad_fn=<AddBackward0>)\n",
      "300 tensor([-0.9670], requires_grad=True) tensor([0.0011], grad_fn=<AddBackward0>)\n",
      "400 tensor([-0.9963], requires_grad=True) tensor([1.4424e-05], grad_fn=<AddBackward0>)\n",
      "500 tensor([-0.9998], requires_grad=True) tensor([5.9605e-08], grad_fn=<AddBackward0>)\n",
      "600 tensor([-1.0000], requires_grad=True) tensor([0.], grad_fn=<AddBackward0>)\n",
      "700 tensor([-1.0000], requires_grad=True) tensor([0.], grad_fn=<AddBackward0>)\n",
      "800 tensor([-1.0000], requires_grad=True) tensor([0.], grad_fn=<AddBackward0>)\n",
      "900 tensor([-1.0000], requires_grad=True) tensor([0.], grad_fn=<AddBackward0>)\n",
      "1000 tensor([-1.0000], requires_grad=True) tensor([0.], grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "torch.manual_seed(0)\n",
    "x = torch.rand(1, requires_grad=True)\n",
    "\n",
    "optimizer = torch.optim.Adam([x], lr=0.01)\n",
    "\n",
    "print(0, x)\n",
    "for i in range(1000):\n",
    "    optimizer.zero_grad()\n",
    "\n",
    "    y = x**2 + 2*x + 1\n",
    "    y.backward(retain_graph=True)\n",
    "    optimizer.step()\n",
    "\n",
    "    if (i + 1) % 100 == 0:\n",
    "        print(i + 1, x, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1.],\n",
      "        [5.]], requires_grad=True)\n",
      "tensor([[0., 1.],\n",
      "        [1., 0.]])\n",
      "tensor([[10.]], grad_fn=<MmBackward>)\n",
      "None\n",
      "tensor([[20.],\n",
      "        [ 4.]])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "x = torch.Tensor([[1],[5]])\n",
    "x.requires_grad = True\n",
    "w = torch.Tensor([[0,1],[1,0]])\n",
    "v = torch.Tensor([[2]])\n",
    "\n",
    "y = x.t() @ w @ x\n",
    "\n",
    "# y = 2 * x1 * x2 // f(x1, x2)\n",
    "# J = [[2*x2, 2*x1] // [[df/dx1, df/dx2]]\n",
    "# J^T = [[2 * x2], [ 2 * x1]] // [[df/dx1], [df/dx2]]\n",
    "# v = [[2]]\n",
    "\n",
    "print(x)\n",
    "print(w)\n",
    "print(y)\n",
    "print(x.grad)\n",
    "y.backward(v)\n",
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1.],\n",
      "        [5.]], requires_grad=True)\n",
      "tensor([[0., 1.],\n",
      "        [1., 0.]])\n",
      "tensor([[10.]], grad_fn=<MmBackward>)\n",
      "tensor([[10.],\n",
      "        [ 2.]])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "x = torch.Tensor([[1],[5]])\n",
    "x.requires_grad = True\n",
    "w = torch.Tensor([[0,1],[1,0]])\n",
    "v = torch.Tensor([[1]])\n",
    "\n",
    "y = x.t() @ w @ x\n",
    "\n",
    "# y = 2 * x1 * x2\n",
    "# J = [[2*x2, 2*x1]\n",
    "# J^T = [[2 * x2], [ 2 * x1]]\n",
    "# v = [[1], [1]]\n",
    "\n",
    "print(x)\n",
    "print(w)\n",
    "print(y)\n",
    "y.backward()\n",
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.2072],\n",
      "        [0.2699]], requires_grad=True)\n",
      "tensor([[1., 2.],\n",
      "        [3., 4.]])\n",
      "tensor([[0.7470],\n",
      "        [1.7012]], grad_fn=<MmBackward>)\n",
      "tensor([[4.],\n",
      "        [6.]])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "x = torch.randn(2, 1, requires_grad=True)\n",
    "w = torch.Tensor([[1,2], [3,4]])\n",
    "v = torch.Tensor([[1], [1]])\n",
    "\n",
    "y = w @ x\n",
    "\n",
    "# y1 = x1 + 2x2\n",
    "# y2 = 3x1 + 4x2\n",
    "# J = [[1,2], [3,4]]\n",
    "# J^T = [[1,3], [2,4]]\n",
    "# v = [[1], [1]]\n",
    "\n",
    "print(x)\n",
    "print(w)\n",
    "print(y)\n",
    "y.backward(v)\n",
    "print(x.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## cross_entropy, log_softmax, nll\n",
    "\n",
    "- https://medium.com/@zhang_yang/understanding-cross-entropy-implementation-in-pytorch-softmax-log-softmax-nll-cross-entropy-416a2b200e34\n",
    "\n",
    "### setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 3])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(0)\n",
    "batch_size, n_classes = 5, 3\n",
    "x = torch.randn(batch_size, n_classes)\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1.5410, -0.2934, -2.1788],\n",
       "        [ 0.5684, -1.0845, -1.3986],\n",
       "        [ 0.4033,  0.8380, -0.7193],\n",
       "        [-0.4033, -0.5966,  0.1820],\n",
       "        [-0.8567,  1.1006, -1.0712]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([2, 0, 1, 0, 1])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target = torch.randint(n_classes, size=(batch_size,), dtype=torch.long)\n",
    "target"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### softmax + nl (negative likelihood)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1.2616)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def softmax(x): return x.exp() / (x.exp().sum(-1)).unsqueeze(-1)\n",
    "def nl(input, target): return -input[range(target.shape[0]), target].log().mean()\n",
    "\n",
    "pred = softmax(x)\n",
    "loss = nl(pred, target)\n",
    "loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1.2616)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred = softmax(x)\n",
    "loss=nl(pred, target)\n",
    "loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### log_softmax + nll (negative log likelihood)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.1689, -2.0033, -3.8886],\n",
      "        [-0.2862, -1.9392, -2.2532],\n",
      "        [-1.0543, -0.6196, -2.1769],\n",
      "        [-1.2865, -1.4797, -0.7011],\n",
      "        [-2.1846, -0.2273, -2.3991]])\n",
      "tensor([[0.8446, 0.1349, 0.0205],\n",
      "        [0.7511, 0.1438, 0.1051],\n",
      "        [0.3484, 0.5382, 0.1134],\n",
      "        [0.2762, 0.2277, 0.4961],\n",
      "        [0.1125, 0.7967, 0.0908]])\n",
      "tensor([2, 0, 1, 0, 1])\n",
      "5\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(1.2616)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def log_softmax(x): return x - x.exp().sum(-1).log().unsqueeze(-1)\n",
    "def nll(input, target): return -input[range(target.shape[0]), target].mean()\n",
    "\n",
    "pred = log_softmax(x)\n",
    "print(pred)\n",
    "print(pred.exp())\n",
    "print(target)\n",
    "print(target.shape[0])\n",
    "loss = nll(pred, target)\n",
    "loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F.log_softmax + F.nll"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.1689, -2.0033, -3.8886],\n",
      "        [-0.2862, -1.9392, -2.2532],\n",
      "        [-1.0543, -0.6196, -2.1769],\n",
      "        [-1.2865, -1.4797, -0.7011],\n",
      "        [-2.1846, -0.2273, -2.3991]])\n",
      "tensor([2, 0, 1, 0, 1])\n",
      "5\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(1.2616)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred = F.log_softmax(x, dim=-1)\n",
    "print(pred)\n",
    "print(target)\n",
    "print(target.shape[0])\n",
    "loss = F.nll_loss(pred, target)\n",
    "loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### F.cross_entropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1.2616)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "F.cross_entropy(x, target)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}