214_PyTorch神经网络基础.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. 层和块"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "① nn.Sequential 定义了一种特殊的Module。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0214, -0.1789, -0.0700, -0.0238, -0.2697,  0.0381,  0.3078, -0.2082,\n",
       "         -0.1502,  0.0433],\n",
       "        [ 0.0200, -0.1466, -0.0633,  0.0031, -0.2042,  0.0993,  0.3137, -0.1206,\n",
       "         -0.1057,  0.0434]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 回顾一下多层感知机\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.nn import functional as F\n",
    "net = nn.Sequential(nn.Linear(20,256),nn.ReLU(),nn.Linear(256,10))\n",
    "X = torch.rand(2,20)\n",
    "net(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. 自定义块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-0.1600,  0.0363,  0.0851,  0.0364,  0.0189,  0.1590,  0.1519,  0.1299,\n",
       "         -0.1382, -0.2075],\n",
       "        [-0.1956,  0.0779, -0.0385, -0.0741,  0.0229,  0.0116,  0.1271,  0.0273,\n",
       "         -0.0867, -0.0511]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class MLP(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()  # 调用父类的__init__函数\n",
    "        self.hidden = nn.Linear(20,256)\n",
    "        self.out = nn.Linear(256,10)\n",
    "        \n",
    "    def forward(self, X):\n",
    "        return self.out(F.relu(self.hidden(X)))\n",
    "    \n",
    "# 实例化多层感知机的层，然后在每次调用正向传播函数调用这些层\n",
    "net = MLP()\n",
    "X = torch.rand(2,20)\n",
    "net(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. 顺序块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Linear(in_features=20, out_features=256, bias=True)\n",
      "ReLU()\n",
      "Linear(in_features=256, out_features=10, bias=True)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "tensor([[-0.0651,  0.0377, -0.0348, -0.0377,  0.1602,  0.0022, -0.0904,  0.1742,\n",
       "         -0.0520,  0.0189],\n",
       "        [-0.0192,  0.1056, -0.0497,  0.0301,  0.2464,  0.0126, -0.1700,  0.4147,\n",
       "          0.0703, -0.0013]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class MySequential(nn.Module):\n",
    "    def __init__(self, *args):\n",
    "        super().__init__()\n",
    "        for block in args:\n",
    "            self._modules[block] = block # block 本身作为它的key，存在_modules里面的为层，以字典的形式\n",
    "            \n",
    "    def forward(self, X):\n",
    "        for block in self._modules.values():\n",
    "            print(block)\n",
    "            X = block(X)\n",
    "        return X\n",
    "    \n",
    "net = MySequential(nn.Linear(20,256),nn.ReLU(),nn.Linear(256,10))\n",
    "X = torch.rand(2,20)\n",
    "net(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. 正向传播"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.3770, grad_fn=<SumBackward0>)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 在正向传播函数中执行代码\n",
    "class FixedHiddenMLP(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.rand_weight = torch.rand((20,20),requires_grad=False)\n",
    "        self.linear = nn.Linear(20,20)\n",
    "    \n",
    "    def forward(self, X):\n",
    "        X = self.linear(X)\n",
    "        X = F.relu(torch.mm(X, self.rand_weight + 1))\n",
    "        X = self.linear(X)\n",
    "        while X.abs().sum() > 1:\n",
    "            X /= 2\n",
    "        return X.sum()\n",
    "    \n",
    "net = FixedHiddenMLP()\n",
    "X = torch.rand(2,20)\n",
    "net(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. 混合组合块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(-0.1488, grad_fn=<SumBackward0>)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 混合代培各种组合块的方法\n",
    "class NestMLP(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.net = nn.Sequential(nn.Linear(20,64),nn.ReLU(),\n",
    "                                nn.Linear(64,32),nn.ReLU())\n",
    "        self.linear = nn.Linear(32,16)\n",
    "        \n",
    "    def forward(self, X):\n",
    "        return self.linear(self.net(X))\n",
    "    \n",
    "chimear = nn.Sequential(NestMLP(),nn.Linear(16,20),FixedHiddenMLP())\n",
    "X = torch.rand(2,20)\n",
    "chimear(X)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. 参数管理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.3941],\n",
      "        [0.4224]], grad_fn=<AddmmBackward0>)\n",
      "OrderedDict([('weight', tensor([[ 4.7564e-02, -5.3226e-02,  1.4919e-04, -2.8679e-01,  1.7408e-01,\n",
      "          3.0859e-01, -1.2281e-01,  5.6171e-02]])), ('bias', tensor([0.3129]))])\n",
      "<class 'torch.nn.parameter.Parameter'>\n",
      "Parameter containing:\n",
      "tensor([0.3129], requires_grad=True)\n",
      "tensor([0.3129])\n",
      "True\n",
      "('weight', torch.Size([8, 4])) ('bias', torch.Size([8]))\n",
      "('0.weight', torch.Size([8, 4])) ('0.bias', torch.Size([8])) ('2.weight', torch.Size([1, 8])) ('2.bias', torch.Size([1]))\n",
      "tensor([0.3129])\n"
     ]
    }
   ],
   "source": [
    "# 首先关注具有单隐藏层的多层感知机\n",
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "net = nn.Sequential(nn.Linear(4,8),nn.ReLU(),nn.Linear(8,1))\n",
    "X = torch.rand(size=(2,4))\n",
    "print(net(X))\n",
    "print(net[2].state_dict()) # 访问参数，net[2]就是最后一个输出层\n",
    "print(type(net[2].bias)) # 目标参数\n",
    "print(net[2].bias)\n",
    "print(net[2].bias.data)\n",
    "print(net[2].weight.grad == None) # 还没进行反向计算，所以grad为None\n",
    "print(*[(name, param.shape) for name, param in net[0].named_parameters()])  # 一次性访问所有参数         \n",
    "print(*[(name, param.shape) for name, param in net.named_parameters()])  # 0是第一层名字，1是ReLU，它没有参数\n",
    "print(net.state_dict()['2.bias'].data) # 通过名字获取参数"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7. 嵌套块"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-0.1750],\n",
      "        [-0.1750]], grad_fn=<AddmmBackward0>)\n",
      "Sequential(\n",
      "  (0): Sequential(\n",
      "    (block0): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "    (block1): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "    (block2): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "    (block3): Sequential(\n",
      "      (0): Linear(in_features=4, out_features=8, bias=True)\n",
      "      (1): ReLU()\n",
      "      (2): Linear(in_features=8, out_features=4, bias=True)\n",
      "      (3): ReLU()\n",
      "    )\n",
      "  )\n",
      "  (1): Linear(in_features=4, out_features=1, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "# 从嵌套块收集参数\n",
    "def block1():\n",
    "    return nn.Sequential(nn.Linear(4,8),nn.ReLU(),nn.Linear(8,4),nn.ReLU())\n",
    "\n",
    "def block2():\n",
    "    net = nn.Sequential()\n",
    "    for i in range(4):\n",
    "        net.add_module(f'block{i}',block1()) # f'block{i}' 可以传一个字符串名字过来，block2可以嵌套四个block1                                      \n",
    "    return net\n",
    "\n",
    "rgnet = nn.Sequential(block2(), nn.Linear(4,1))\n",
    "print(rgnet(X))\n",
    "print(rgnet)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 8 内置初始化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 0.0012, -0.0112, -0.0153,  0.0218])\n",
      "tensor(0.)\n"
     ]
    }
   ],
   "source": [
    "net = nn.Sequential(nn.Linear(4,8),nn.ReLU(),nn.Linear(8,1))\n",
    "\n",
    "def init_normal(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.normal_(m.weight, mean=0, std=0.01) # 下划线表示把m.weight的值替换掉   \n",
    "        nn.init.zeros_(m.bias)\n",
    "        \n",
    "net.apply(init_normal) # 会递归调用 直到所有层都初始化\n",
    "print(net[0].weight.data[0])\n",
    "print(net[0].bias.data[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1., 1., 1., 1.])\n",
      "tensor(0.)\n"
     ]
    }
   ],
   "source": [
    "net = nn.Sequential(nn.Linear(4,8),nn.ReLU(),nn.Linear(8,1))\n",
    "\n",
    "def init_constant(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.constant_(m.weight,1)\n",
    "        nn.init.zeros_(m.bias)\n",
    "        \n",
    "net.apply(init_constant)\n",
    "print(net[0].weight.data[0]) \n",
    "print(net[0].bias.data[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 0.0479, -0.1771,  0.5267, -0.0020])\n",
      "tensor([[42., 42., 42., 42., 42., 42., 42., 42.]])\n"
     ]
    }
   ],
   "source": [
    "# 对某些块应用不同的初始化\n",
    "def xavier(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.xavier_uniform_(m.weight)\n",
    "        \n",
    "def init_42(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        nn.init.constant_(m.weight, 42)\n",
    "        \n",
    "net[0].apply(xavier)\n",
    "net[2].apply(init_42)\n",
    "print(net[0].weight.data[0])\n",
    "print(net[2].weight.data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 9. 参数替换"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Init weight torch.Size([8, 4])\n",
      "Init weight torch.Size([1, 8])\n",
      "tensor([[ 0.0000,  7.1240,  0.0000,  5.1135],\n",
      "        [-8.6745, -7.3974,  0.0000, -0.0000]], grad_fn=<SliceBackward0>)\n",
      "tensor([42.0000,  8.1240,  1.0000,  6.1135])\n"
     ]
    }
   ],
   "source": [
    "# 自定义初始化\n",
    "def my_init(m):\n",
    "    if type(m) == nn.Linear:\n",
    "        print(\"Init\",*[(name, param.shape) for name, param in m.named_parameters()][0])  # 打印名字是啥，形状是啥       \n",
    "        nn.init.uniform_(m.weight, -10, 10)\n",
    "        m.weight.data *= m.weight.data.abs() >=  5 # 这里*=的代码相当于先计算一个布尔矩阵(先判断>=)，然后再用布尔矩阵的对应元素去乘以原始矩阵的每个元素。保留绝对值大于5的权重，不是的话就设为0\n",
    "\n",
    "net.apply(my_init)\n",
    "print(net[0].weight[:2])\n",
    "net[0].weight.data[:] += 1 # 参数替换\n",
    "net[0].weight.data[0,0] = 42\n",
    "print(net[0].weight.data[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 10. 参数绑定"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([True, True, True, True, True, True, True, True])\n",
      "tensor([True, True, True, True, True, True, True, True])\n"
     ]
    }
   ],
   "source": [
    "# 参数绑定\n",
    "shared = nn.Linear(8,8)\n",
    "net = nn.Sequential(nn.Linear(4,8),nn.ReLU(),shared,nn.ReLU(),shared,nn.ReLU(),nn.Linear(8,1))  # 第2个隐藏层和第3个隐藏层是share权重的，第一个和第四个是自己的  \n",
    "net(X)\n",
    "print(net[2].weight.data[0] == net[4].weight.data[0])\n",
    "net[2].weight.data[0,0] = 100\n",
    "print(net[2].weight.data[0] == net[4].weight.data[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 11. 自定义层"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([-2., -1.,  0.,  1.,  2.])\n",
      "tensor(-6.2864e-09, grad_fn=<MeanBackward0>)\n",
      "Parameter containing:\n",
      "tensor([[-2.8449,  0.1887,  0.7945],\n",
      "        [ 0.4226,  1.6180, -0.5880],\n",
      "        [-0.4794, -0.0817, -0.3648],\n",
      "        [-0.1979,  0.8702, -0.3515],\n",
      "        [-1.4943,  0.3618,  0.2969]], requires_grad=True)\n",
      "tensor([[0.0000, 0.0000, 1.3957],\n",
      "        [0.8225, 0.0000, 0.9089]])\n",
      "tensor([[0.],\n",
      "        [0.]])\n"
     ]
    }
   ],
   "source": [
    "# 构造一个没有任何参数的自定义层\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from torch import nn\n",
    "\n",
    "class CenteredLayer(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "    def forward(self, X):\n",
    "        return X - X.mean()\n",
    "    \n",
    "layer = CenteredLayer()\n",
    "print(layer(torch.FloatTensor([1,2,3,4,5])))\n",
    "\n",
    "# 将层作为组件合并到构建更复杂的模型中\n",
    "net = nn.Sequential(nn.Linear(8,128),CenteredLayer())\n",
    "Y = net(torch.rand(4,8))\n",
    "print(Y.mean())\n",
    "\n",
    "# 带参数的图层\n",
    "class MyLinear(nn.Module):\n",
    "    def __init__(self, in_units, units):\n",
    "        super().__init__()\n",
    "        self.weight = nn.Parameter(torch.randn(in_units,units)) # nn.Parameter使得这些参数加上了梯度    \n",
    "        self.bias = nn.Parameter(torch.randn(units,))\n",
    "\n",
    "    def forward(self, X):\n",
    "        linear = torch.matmul(X, self.weight.data) + self.bias.data\n",
    "        return F.relu(linear)\n",
    "    \n",
    "dense = MyLinear(5,3)\n",
    "print(dense.weight)\n",
    "\n",
    "# 使用自定义层直接执行正向传播计算\n",
    "print(dense(torch.rand(2,5)))\n",
    "# 使用自定义层构建模型\n",
    "net = nn.Sequential(MyLinear(64,8),MyLinear(8,1))\n",
    "print(net(torch.rand(2,64)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 12. 读写文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0, 1, 2, 3])\n",
      "tensor([0, 1, 2, 3])\n",
      "tensor([0., 0., 0., 0.])\n",
      "{'x': tensor([0, 1, 2, 3]), 'y': tensor([0., 0., 0., 0.])}\n"
     ]
    }
   ],
   "source": [
    "# 加载和保存张量\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.nn import functional as F\n",
    "\n",
    "x = torch.arange(4)\n",
    "torch.save(x, 'x-file')\n",
    "x2 = torch.load(\"x-file\")\n",
    "print(x2)\n",
    "\n",
    "#存储一个张量列表，然后把它们读回内存\n",
    "y = torch.zeros(4)\n",
    "torch.save([x,y],'x-files')\n",
    "x2, y2 = torch.load('x-files')\n",
    "print(x2)\n",
    "print(y2)\n",
    "\n",
    "# 写入或读取从字符串映射到张量的字典\n",
    "mydict = {'x':x,'y':y}\n",
    "torch.save(mydict,'mydict')\n",
    "mydict2 = torch.load('mydict')\n",
    "print(mydict2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MLP(\n",
      "  (hidden): Linear(in_features=20, out_features=256, bias=True)\n",
      "  (output): Linear(in_features=256, out_features=10, bias=True)\n",
      ")\n",
      "tensor([[True, True, True, True, True, True, True, True, True, True],\n",
      "        [True, True, True, True, True, True, True, True, True, True]])\n"
     ]
    }
   ],
   "source": [
    "# 加载和保存模型参数\n",
    "class MLP(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.hidden = nn.Linear(20,256)\n",
    "        self.output = nn.Linear(256,10)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        return self.output(F.relu(self.hidden(x)))\n",
    "    \n",
    "net = MLP()\n",
    "X = torch.randn(size=(2,20))\n",
    "Y = net(X)\n",
    "\n",
    "# 将模型的参数存储为一个叫做\"mlp.params\"的文件\n",
    "torch.save(net.state_dict(),'mlp.params')\n",
    "\n",
    "# 实例化了原始多层感知机模型的一个备份。直接读取文件中存储的参数\n",
    "clone = MLP() # 必须要先声明一下，才能导入参数\n",
    "clone.load_state_dict(torch.load(\"mlp.params\"))\n",
    "print(clone.eval()) # eval()是进入测试模式\n",
    "\n",
    "Y_clone = clone(X)\n",
    "print(Y_clone == Y)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.6.3",
   "language": "python",
   "name": "python3.6.3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": false,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "2041.23px",
    "left": "45.9792px",
    "top": "56px",
    "width": "301.875px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}