torch_toy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tensor related torch examples\n",
    "Tensor 관련된 기본적인 메소드들 모아놓음"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.nn.utils.rnn import pad_sequence\n",
    "import torch.optim as optim\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## indexing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.arange(0, 10).reshape(2, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([2, 5])"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 1, 2, 3, 4],\n",
       "        [5, 6, 7, 8, 9]])"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[:2] # 0번째 차원에서 0~1번째 텐서 보자(=> 전체 텐서)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(8)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[1, 3] # 0번째 차원에서 2번째(2행), 1번째 차원에서 4번째(4열) 보자"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([3, 9])"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[[0, 1], [3, 4]] # 1열 4행과 2행 5열 뽑자"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([3, 9])"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx = [[0, 1], [3, 4]] # 텐서를 인덱싱하는 건 일단 리스트라고 볼 수 있음. 그럼 list를 idx로 할당하고 인덱싱해도 똑같은 결과!\n",
    "x[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "idx shape torch.Size([2, 2])\n"
     ]
    },
    {
     "ename": "IndexError",
     "evalue": "index 3 is out of bounds for dimension 0 with size 2",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-105-ac6d42d081a0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0midx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m4\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# 근데 같은 내용이라도 이게 텐서로 들어가있으면?\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"idx shape\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0midx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# 그냥 행차원에서 인덱싱하는게 되고, 이때 만들어지는 텐서는 인덱싱한 shape을 따라간다.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;31m# x의 shape은 2행 5열인데 3, 4 행을 인덱싱하려고 해서 인덱스 에러\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mIndexError\u001b[0m: index 3 is out of bounds for dimension 0 with size 2"
     ]
    }
   ],
   "source": [
    "idx = torch.tensor([[0, 1], [3, 4]]) # 근데 같은 내용이라도 이게 텐서로 들어가있으면?\n",
    "print(\"idx shape\", idx.shape) # 그냥 행차원에서 인덱싱하는게 되고, 이때 만들어지는 텐서는 인덱싱한 shape을 따라간다.\n",
    "x[idx] # x의 shape은 2행 5열인데 3, 4 행을 인덱싱하려고 해서 인덱스 에러"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 1, 2, 3, 4])"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 1, 2, 3, 4])"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[torch.tensor(0)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1)"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[0, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 1, 2, 3, 4],\n",
       "        [5, 6, 7, 8, 9]])"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[torch.tensor([0, 1])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[[0, 1, 2, 3, 4],\n",
      "          [5, 6, 7, 8, 9]]]])\n",
      "torch.Size([1, 1, 2, 5])\n"
     ]
    }
   ],
   "source": [
    "_ = x[torch.tensor([[[0, 1]]])] \n",
    "print(_) \n",
    "print(_.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 2])\n",
      "tensor([[[0, 1, 2, 3, 4],\n",
      "         [5, 6, 7, 8, 9]],\n",
      "\n",
      "        [[5, 6, 7, 8, 9],\n",
      "         [0, 1, 2, 3, 4]]])\n",
      "torch.Size([2, 2, 5])\n"
     ]
    }
   ],
   "source": [
    "idx = torch.tensor([[0, 1], [1, 0]])\n",
    "_ = x[idx]\n",
    "print(idx.shape)\n",
    "print(_)\n",
    "print(_.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1, 0],\n",
       "        [6, 5]])"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[:, idx[1]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "그러면 원래 목표대로 list처럼 인덱싱을 하려면 어떻게 할까?<br>\n",
    "아래와 같이 ,를 사용하게 명시적으로 주면 된다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([3, 9])"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx = torch.tensor([[0, 1], [3, 4]]) \n",
    "x[idx[0, :], idx[1, :]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## .nonzero() \n",
    "텐서에서 요소가 0이 아닌 index 나옴"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 1],\n",
       "        [1, 0],\n",
       "        [1, 1],\n",
       "        [1, 2]])"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.tensor([[0, 1, 0], [1, 1, 1]]).nonzero()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## scatter\n",
    "src, idx, tgt 텐서 세개가 있는데 src에 있는 idx만 가져와서 tgt에 적용하고 싶을 때 쓰는 메소드. 원핫으로 만들 때 많이 쓰는듯? 근데 one-hot은 메소드 따로 있긴 함..\n",
    "https://yuyangyy.medium.com/understand-torch-scatter-b0fd6275331c<br>\n",
    "Writes all values from the tensor src into self at the indices specified in the index tensor. For each value in src, its output index is specified by its index in src for dimension != dim and by the corresponding value in index for dimension = dim."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 1,  2,  3,  4,  5],\n",
       "        [ 6,  7,  8,  9, 10]])"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "src = torch.arange(1, 11).reshape((2, 5))\n",
    "src"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 1, 2, 0]])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index = torch.tensor([[0, 1, 2, 0]])\n",
    "index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([3, 5]), torch.Size([2, 5]), torch.Size([1, 4]))"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.zeros(3, 5, dtype=src.dtype).shape, src.shape, index.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1, 0, 0, 4, 0],\n",
       "        [0, 2, 0, 0, 0],\n",
       "        [0, 0, 3, 0, 0]])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.zeros(3, 5, dtype=src.dtype).scatter_(0, index, src)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[4, 2, 3, 0, 0],\n",
       "        [0, 0, 0, 0, 0],\n",
       "        [0, 0, 0, 0, 0]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# [1, 2, 3, 0, 0] -> [4, 2, 3, 0, 0]으로 된듯\n",
    "torch.zeros(3, 5, dtype=src.dtype).scatter_(1, index, src)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1.,  2.,  3.,  4.,  5.],\n",
      "        [ 6.,  7.,  8.,  9., 10.]])\n"
     ]
    }
   ],
   "source": [
    "src = torch.from_numpy(np.arange(1, 11)).float().view(2, 5)\n",
    "print(src)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0.],\n",
      "        [0., 0., 0., 0., 0.]])\n"
     ]
    }
   ],
   "source": [
    "input_tensor = torch.zeros(3, 5)\n",
    "print(input_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0, 1, 2, 0, 0],\n",
      "        [2, 0, 0, 1, 2]])\n"
     ]
    }
   ],
   "source": [
    "index_tensor = torch.tensor([[0, 1, 2, 0, 0], [2, 0, 0, 1, 2]])\n",
    "print(index_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 1.,  7.,  8.,  4.,  5.],\n",
      "        [ 0.,  2.,  0.,  9.,  0.],\n",
      "        [ 6.,  0.,  3.,  0., 10.]])\n"
     ]
    }
   ],
   "source": [
    "## try to manually work out the result \n",
    "dim = 0\n",
    "print(input_tensor.scatter_(dim, index_tensor, src))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "one-hot encoding할 때 쓰인다고 함\n",
    "https://aigong.tistory.com/35"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = torch.Tensor([3, 6, 7]).long()\n",
    "one_hot = torch.zeros(5, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "label = label.unsqueeze(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([3, 1])"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0., 0., 1., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 1., 0., 0.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "        [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.]])"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "one_hot.scatter_(1, label, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# einsum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 0,  1,  2,  3],\n",
      "        [ 4,  5,  6,  7],\n",
      "        [ 8,  9, 10, 11],\n",
      "        [12, 13, 14, 15]])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor(30)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# trace\n",
    "x = torch.arange(0, 16).reshape(4, 4)\n",
    "print(x)\n",
    "torch.einsum(\"ii\", x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## where\n",
    "mask_fill의 더 심화된 버전. src, tgt이 있을 때 src을 어떤 조건을 걸고 (boolean) True인 경우를 tgt element로 바꿔치기!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    ">>> x = torch.randn(3, 2)\n",
    ">>> y = torch.ones(3, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-1.6403,  0.8030],\n",
       "        [ 2.4286, -0.2958],\n",
       "        [ 1.3324,  0.3157]])"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ">>> x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.0000, 0.8030],\n",
       "        [2.4286, 1.0000],\n",
       "        [1.3324, 0.3157]])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    ">>> torch.where(x > 0, x, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## repeat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1, 2, 3, 1, 2, 3],\n",
       "        [1, 2, 3, 1, 2, 3],\n",
       "        [1, 2, 3, 1, 2, 3],\n",
       "        [1, 2, 3, 1, 2, 3]])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    ">>> x = torch.tensor([1, 2, 3])\n",
    ">>> x.repeat(4, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 1, 2, 3, 4, 5, 6, 7, 8, 9],\n",
       "        [0, 1, 2, 3, 4, 5, 6, 7, 8, 9],\n",
       "        [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "torch.arange(0, 10).unsqueeze(0).repeat(3, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## permute\n",
    "(*dims) → Tensor\n",
    "Returns a view of the original tensor with its dimensions permuted.\n",
    "\n",
    "Parameters\n",
    "*dims (int...) – The desired ordering of dimensions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([5, 2, 3])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    ">>> x = torch.randn(2, 3, 5)\n",
    ">>> x.size()\n",
    "torch.Size([2, 3, 5])\n",
    ">>> x.permute(2, 0, 1).size()\n",
    "torch.Size([5, 2, 3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## product\n",
    "- `*` : elementwise product \n",
    "- `@` : inner product or matmul \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3])\n"
     ]
    }
   ],
   "source": [
    "v = torch.Tensor([1, 2, 3])\n",
    "print(v.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1., 4., 9.])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "v * v # hadamard product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1., 4., 9.])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "v * v.T # hadamard product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([1., 4., 9.])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "v.T * v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(14.)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "v @ v # inner product"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(14.)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "v.T @ v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(14.)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "v @ v.T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 3])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "v.unsqueeze_(0)\n",
    "v.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2., 3., 4.],\n",
       "        [3., 4., 5.],\n",
       "        [4., 5., 6.]])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "v + v.T # broad casting "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 2., 3.],\n",
       "        [2., 4., 6.],\n",
       "        [3., 6., 9.]])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "v * v.T # (1 x 3) * (3 x 1) -> broad casting "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 2., 3.],\n",
       "        [2., 4., 6.],\n",
       "        [3., 6., 9.]])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "v.T * v # (3 x 1) * (1 x 3) -> broad casting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[14.]])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "v @ v.T # 1 x 3 @ 3 x 1 -> 1 x 1 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "mat1 and mat2 shapes cannot be multiplied (1x3 and 1x3)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)\n",
      "\u001b[1;32m/home/jy.nam/jy.nam/torch_study/torch_toy.ipynb 셀 27\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Bserver006/home/jy.nam/jy.nam/torch_study/torch_toy.ipynb#Y141sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m v \u001b[39m@\u001b[39;49m v\n",
      "\n",
      "\u001b[0;31mRuntimeError\u001b[0m: mat1 and mat2 shapes cannot be multiplied (1x3 and 1x3)"
     ]
    }
   ],
   "source": [
    "v @ v # 1 x 3 @ 1 x 3 -> cannot multiply!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## bmm\n",
    "Performs a batch matrix-matrix product of matrices stored in input and mat2.\n",
    "\n",
    "input and mat2 must be 3-D tensors each containing the same number of matrices."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([10, 3, 5])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "input = torch.randn(10, 3, 4)\n",
    "mat2 = torch.randn(10, 4, 5)\n",
    "res = torch.bmm(input, mat2)\n",
    "res.size()\n",
    "torch.Size([10, 3, 5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## matmul\n",
    "https://pytorch.org/docs/stable/generated/torch.matmul.html#torch.matmul\n",
    "\n",
    "broadcasting된다는 것이 bmm과의 차이"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([10, 3, 5])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "input = torch.randn(10, 3, 4)\n",
    "mat2 = torch.randn(10, 4, 5)\n",
    "res = input.matmul(mat2)\n",
    "res.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([10, 3, 5])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    ">>> # batched matrix x broadcasted matrix\n",
    ">>> tensor1 = torch.randn(10, 3, 4)\n",
    ">>> tensor2 = torch.randn(4, 5)\n",
    ">>> torch.matmul(tensor1, tensor2).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "batch2 must be a 3D tensor",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)\n",
      "\u001b[1;32m/home/jy.nam/jy.nam/torch_study/torch_toy.ipynb 셀 23\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Bserver006/home/jy.nam/jy.nam/torch_study/torch_toy.ipynb#X31sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m torch\u001b[39m.\u001b[39;49mbmm(tensor1, tensor2)\n",
      "\n",
      "\u001b[0;31mRuntimeError\u001b[0m: batch2 must be a 3D tensor"
     ]
    }
   ],
   "source": [
    "torch.bmm(tensor1, tensor2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## tril\n",
    "하부 삼각형만 남기고 상부 삼각형은 0으로.. 뭐에 쓰냐 이거"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 4.4842e-44,  0.0000e+00,  0.0000e+00,  0.0000e+00],\n",
       "        [-2.1006e+23,  3.0787e-41,  0.0000e+00,  0.0000e+00],\n",
       "        [ 7.0065e-45,  0.0000e+00,  2.3354e-07,  0.0000e+00]])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "torch.tril(torch.Tensor(3, 4))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## masked_fill"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.arange(10)\n",
    "mask = x < 5 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ True,  True,  True,  True,  True, False, False, False, False, False])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([3, 3, 3, 3, 3, 5, 6, 7, 8, 9])"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x.masked_fill(mask, 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## view vs. transpose\n",
    "웬만하면 transpose를 쓰는게 맞음. 다 펴주고 싶을 때 view(-1) 처럼 쓰는거 말고는 쓸 때 유의하자."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9],\n",
       "         [10, 11, 12, 13, 14, 15, 16, 17, 18, 19],\n",
       "         [20, 21, 22, 23, 24, 25, 26, 27, 28, 29],\n",
       "         [30, 31, 32, 33, 34, 35, 36, 37, 38, 39],\n",
       "         [40, 41, 42, 43, 44, 45, 46, 47, 48, 49]],\n",
       "\n",
       "        [[50, 51, 52, 53, 54, 55, 56, 57, 58, 59],\n",
       "         [60, 61, 62, 63, 64, 65, 66, 67, 68, 69],\n",
       "         [70, 71, 72, 73, 74, 75, 76, 77, 78, 79],\n",
       "         [80, 81, 82, 83, 84, 85, 86, 87, 88, 89],\n",
       "         [90, 91, 92, 93, 94, 95, 96, 97, 98, 99]]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.arange(100)\n",
    "x.view(2, 5, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9],\n",
       "         [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]],\n",
       "\n",
       "        [[20, 21, 22, 23, 24, 25, 26, 27, 28, 29],\n",
       "         [30, 31, 32, 33, 34, 35, 36, 37, 38, 39]],\n",
       "\n",
       "        [[40, 41, 42, 43, 44, 45, 46, 47, 48, 49],\n",
       "         [50, 51, 52, 53, 54, 55, 56, 57, 58, 59]],\n",
       "\n",
       "        [[60, 61, 62, 63, 64, 65, 66, 67, 68, 69],\n",
       "         [70, 71, 72, 73, 74, 75, 76, 77, 78, 79]],\n",
       "\n",
       "        [[80, 81, 82, 83, 84, 85, 86, 87, 88, 89],\n",
       "         [90, 91, 92, 93, 94, 95, 96, 97, 98, 99]]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.arange(100)\n",
    "x.view(5, 2, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = torch.arange(10)\n",
    "xx = x.view(5, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 1],\n",
       "        [2, 3],\n",
       "        [4, 5],\n",
       "        [6, 7],\n",
       "        [8, 9]])"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 1, 2, 3, 4],\n",
       "        [5, 6, 7, 8, 9]])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xx.view(2, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 2, 4, 6, 8],\n",
       "        [1, 3, 5, 7, 9]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xx.transpose(1, 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## view vs. reshape\n",
    "view는 contiguous한 텐서에 대해서만 가능하고 reshape은 contiguous한 tensor가 아니어도 연산 가능\n",
    "-> view가 reshape보다 더 안전한 메소드인듯 하다\n",
    "\n",
    "https://stackoverflow.com/questions/26998223/what-is-the-difference-between-contiguous-and-non-contiguous-arrays/26999092#26999092"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 1, 2, 3, 4],\n",
       "        [5, 6, 7, 8, 9]])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.reshape(2, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 1, 2, 3, 4],\n",
       "        [5, 6, 7, 8, 9]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.view(2, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = x.view(2, 5).T\n",
    "x.is_contiguous()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "view size is not compatible with input tensor's size and stride (at least one dimension spans across two contiguous subspaces). Use .reshape(...) instead.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-19-dd9df8a4b3e0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# error가 난다\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m: view size is not compatible with input tensor's size and stride (at least one dimension spans across two contiguous subspaces). Use .reshape(...) instead."
     ]
    }
   ],
   "source": [
    "x.view(2, 5) # error가 난다"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 5, 1, 6, 2],\n",
       "        [7, 3, 8, 4, 9]])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.contiguous().view(2, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0, 5, 1, 6, 2],\n",
       "        [7, 3, 8, 4, 9]])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x.reshape(2, 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## view vs flatten\n",
    "view는 원래 순서대로가 아니면 .view(-1) 못하고 flatten은 상관없이 할 수 있음.\n",
    "flatten이 더 편할 때는 4개 이상 차원이 있는데 앞 두개를 flatten하고 싶을 때 편할듯?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])\n"
     ]
    },
    {
     "ename": "RuntimeError",
     "evalue": "view size is not compatible with input tensor's size and stride (at least one dimension spans across two contiguous subspaces). Use .reshape(...) instead.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-37-9027cddecb87>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m: view size is not compatible with input tensor's size and stride (at least one dimension spans across two contiguous subspaces). Use .reshape(...) instead."
     ]
    }
   ],
   "source": [
    "x = torch.arange(0, 10).reshape(2, 5)\n",
    "print(x.view(-1))\n",
    "x.T.view(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([0, 5, 1, 6, 2, 7, 3, 8, 4, 9])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(x.flatten(0, 1))\n",
    "x.T.flatten(0, 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.8 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "vscode": {
   "interpreter": {
    "hash": "4de80974f87294fd1fdcc19e5ba4ef608a8b1a6d8ff488c40fc196dc4f2301bf"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}