6.0.2 - MLP 3.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"DL-with-pytorch - 3 [MLP].ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"metadata":{"id":"bIAYbYajk1w9","colab_type":"text"},"cell_type":"markdown","source":["# Multi Layer Perceptrons\n","- CIFAR-10 image classification with deeper MLP\n","  - Last time, we have attempted to classify images in the CIFAR-10 dataset with MLP. However, we were able to obtain accuracy of only 0.369.\n","  - Here, we attempt to gain higher accuracy by changing the MLP architecture and hyperparameters."]},{"metadata":{"id":"GVU5-yp3N89I","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":336},"outputId":"54e7c80a-d066-4716-bfcc-dd50dab9977a","executionInfo":{"status":"ok","timestamp":1545850085005,"user_tz":420,"elapsed":64720,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}}},"cell_type":"code","source":["!pip3 install torch torchvision"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Collecting torch\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7e/60/66415660aa46b23b5e1b72bc762e816736ce8d7260213e22365af51e8f9c/torch-1.0.0-cp36-cp36m-manylinux1_x86_64.whl (591.8MB)\n","\u001b[K    100% |████████████████████████████████| 591.8MB 26kB/s \n","tcmalloc: large alloc 1073750016 bytes == 0x60d7e000 @  0x7f87578fc2a4 0x591a07 0x5b5d56 0x502e9a 0x506859 0x502209 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x507641 0x502209 0x502f3d 0x506859 0x504c28 0x502540 0x502f3d 0x507641 0x504c28 0x502540 0x502f3d 0x507641\n","\u001b[?25hCollecting torchvision\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ca/0d/f00b2885711e08bd71242ebe7b96561e6f6d01fdb4b9dcf4d37e2e13c5e1/torchvision-0.2.1-py2.py3-none-any.whl (54kB)\n","\u001b[K    100% |████████████████████████████████| 61kB 23.5MB/s \n","\u001b[?25hCollecting pillow>=4.1.1 (from torchvision)\n","\u001b[?25l  Downloading https://files.pythonhosted.org/packages/62/94/5430ebaa83f91cc7a9f687ff5238e26164a779cca2ef9903232268b0a318/Pillow-5.3.0-cp36-cp36m-manylinux1_x86_64.whl (2.0MB)\n","\u001b[K    100% |████████████████████████████████| 2.0MB 4.2MB/s \n","\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from torchvision) (1.11.0)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torchvision) (1.14.6)\n","Installing collected packages: torch, pillow, torchvision\n","  Found existing installation: Pillow 4.0.0\n","    Uninstalling Pillow-4.0.0:\n","      Successfully uninstalled Pillow-4.0.0\n","Successfully installed pillow-5.3.0 torch-1.0.0 torchvision-0.2.1\n"],"name":"stdout"}]},{"metadata":{"id":"8yy37hEYOEiQ","colab_type":"code","outputId":"1b368262-43ed-4ed4-af57-992d8f166a8e","executionInfo":{"status":"ok","timestamp":1545795541495,"user_tz":420,"elapsed":757,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":["import numpy as np\n","import pandas as pd\n","import torch, torchvision\n","import torch.nn as nn\n","import torch.nn.functional as F\n","torch.__version__"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'1.0.0'"]},"metadata":{"tags":[]},"execution_count":1}]},{"metadata":{"id":"ewrw93tt2BfV","colab_type":"text"},"cell_type":"markdown","source":["## 1. Import & process dataset\n","- CIFAR10 dataset can be downloaded by ```torchvision```\n","  - [torchvision.datasets](https://pytorch.org/docs/stable/torchvision/datasets.html)"]},{"metadata":{"id":"W5anlYa01w3w","colab_type":"code","outputId":"ca1e77b2-1a79-4e0e-a2fa-d12a8267ecd3","executionInfo":{"status":"ok","timestamp":1545795548014,"user_tz":420,"elapsed":2564,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"colab":{"base_uri":"https://localhost:8080/","height":52}},"cell_type":"code","source":["from torchvision import datasets\n","import torchvision.transforms as transforms\n","\n","train_dataset = datasets.CIFAR10(root = \"/\", train = True, download = True, transform = transforms.ToTensor())\n","test_dataset = datasets.CIFAR10(root = \"/\", train = False, download = True, transform = transforms.ToTensor())"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Files already downloaded and verified\n","Files already downloaded and verified\n"],"name":"stdout"}]},{"metadata":{"id":"9nznVMdo5edZ","colab_type":"text"},"cell_type":"markdown","source":["## 2. Creating MLP model and training\n","\n","- Create and train MLP with more hidden layers and advanced settings\n","\n","![](http://cs231n.github.io/assets/nn1/neural_net.jpeg)"]},{"metadata":{"id":"YEoNQr10kGUW","colab_type":"code","colab":{}},"cell_type":"code","source":["# create data loaders \n","train_loader = torch.utils.data.DataLoader(train_dataset, batch_size = 128, shuffle = True)\n","test_loader = torch.utils.data.DataLoader(test_dataset, batch_size = 128, shuffle = False)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"AQawpMRPI7jm","colab_type":"code","colab":{}},"cell_type":"code","source":["# create MLP with one hidden layer\n","class net(nn.Module):\n","  def __init__(self, input_dim, hidden_dim, output_dim):\n","    super(net, self).__init__()\n","    self.input_dim = input_dim\n","    self.dense1 = nn.Linear(input_dim, hidden_dim[0])      # first hidden layer\n","    self.dense2 = nn.Linear(hidden_dim[0], hidden_dim[1])  # second hidden layer\n","    self.dense3 = nn.Linear(hidden_dim[1], hidden_dim[2])  # third hidden layer\n","    self.dense4 = nn.Linear(hidden_dim[2], output_dim)     \n","    self.relu = nn.ReLU()\n","     \n","  def forward(self, x):\n","    x = x.view(x.size(0), self.input_dim)\n","    x = self.dense1(x)\n","    x = self.relu(x)\n","    x = self.dense2(x)\n","    x = self.relu(x)\n","    x = self.dense3(x)\n","    x = self.relu(x)\n","    x = self.dense4(x)\n","    return x"],"execution_count":0,"outputs":[]},{"metadata":{"id":"rP0Gt5E9ajmd","colab_type":"code","colab":{}},"cell_type":"code","source":["# hyperparameters\n","DEVICE = torch.device('cuda')\n","INPUT_DIM = 32 * 32 * 3\n","HIDDEN_DIM = (100, 100, 100)    # 3 hidden dimensions for 3 layers\n","OUTPUT_DIM = 10\n","LEARNING_RATE = 1e-3\n","NUM_EPOCHS = 30              # increase number of epochs"],"execution_count":0,"outputs":[]},{"metadata":{"id":"cPBm8qDrSWsi","colab_type":"code","colab":{}},"cell_type":"code","source":["model = net(INPUT_DIM, HIDDEN_DIM, OUTPUT_DIM).to(DEVICE)\n","criterion = nn.CrossEntropyLoss()   # do not need softmax layer when using CEloss criterion\n","optimizer = torch.optim.Adam(model.parameters(), lr = LEARNING_RATE)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"SEBtAPYCFeic","colab_type":"code","outputId":"ff60d729-d864-4e56-a2fa-ae44100f747e","executionInfo":{"status":"ok","timestamp":1545796827881,"user_tz":420,"elapsed":292007,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"colab":{"base_uri":"https://localhost:8080/","height":545}},"cell_type":"code","source":["# training for NUM_EPOCHS\n","for i in range(NUM_EPOCHS):\n","  temp_loss = []\n","  for (x, y) in train_loader:\n","    x, y = x.float().to(DEVICE), y.to(DEVICE)\n","    outputs = model(x)\n","    loss = criterion(outputs, y)\n","    temp_loss.append(loss.item())\n","    \n","    optimizer.zero_grad()\n","    loss.backward()\n","    optimizer.step()\n","    \n","  print(\"Loss at {}th epoch: {}\".format(i, np.mean(temp_loss)))"],"execution_count":0,"outputs":[{"output_type":"stream","text":["Loss at 0th epoch: 1.89627995271512\n","Loss at 1th epoch: 1.693795374592247\n","Loss at 2th epoch: 1.6212971856831895\n","Loss at 3th epoch: 1.5668071813290687\n","Loss at 4th epoch: 1.5262251299665408\n","Loss at 5th epoch: 1.4933200495322343\n","Loss at 6th epoch: 1.4687112606394932\n","Loss at 7th epoch: 1.4454451488411946\n","Loss at 8th epoch: 1.4289868929806877\n","Loss at 9th epoch: 1.408607015829257\n","Loss at 10th epoch: 1.394239951582516\n","Loss at 11th epoch: 1.3733605451291175\n","Loss at 12th epoch: 1.3631285389366052\n","Loss at 13th epoch: 1.343611014011266\n","Loss at 14th epoch: 1.3399841968360764\n","Loss at 15th epoch: 1.3238185805737819\n","Loss at 16th epoch: 1.3146519923149167\n","Loss at 17th epoch: 1.3017306763802647\n","Loss at 18th epoch: 1.2912272934413627\n","Loss at 19th epoch: 1.2836608034570505\n","Loss at 20th epoch: 1.2702278880512012\n","Loss at 21th epoch: 1.2586160532348907\n","Loss at 22th epoch: 1.2533685801279209\n","Loss at 23th epoch: 1.2392126495575966\n","Loss at 24th epoch: 1.2397530463040638\n","Loss at 25th epoch: 1.2295762690741692\n","Loss at 26th epoch: 1.2183618973892973\n","Loss at 27th epoch: 1.2042760227037512\n","Loss at 28th epoch: 1.2044923659175863\n","Loss at 29th epoch: 1.2012101366086994\n"],"name":"stdout"}]},{"metadata":{"id":"qpAJUiHm529m","colab_type":"text"},"cell_type":"markdown","source":["## 3. Evaluation\n","- Evaluate the trained MLP model with ROC AUC score \n","  - Store probability of each instance to a list and compare it with true y label"]},{"metadata":{"id":"txXH3dknFpSx","colab_type":"code","outputId":"0ceabe35-2158-40ac-82e8-428937e4169a","executionInfo":{"status":"ok","timestamp":1545798230878,"user_tz":420,"elapsed":2322,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"colab":{"base_uri":"https://localhost:8080/","height":72}},"cell_type":"code","source":["y_pred, y_true = [], []\n","with torch.no_grad():\n","  for x, y in test_loader:\n","    x, y = x.float().to(DEVICE), y.to(DEVICE)\n","    outputs = F.softmax(model(x)).max(1)[-1]       # predicted label\n","    y_true += list(y.cpu().numpy())                # true label\n","    y_pred += list(outputs.cpu().numpy())   "],"execution_count":0,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:5: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n","  \"\"\"\n"],"name":"stderr"}]},{"metadata":{"id":"HV1s3xf5Frkl","colab_type":"code","outputId":"f5ca3a29-ffba-4b9a-993f-5f9c4106e0ae","executionInfo":{"status":"ok","timestamp":1545798232470,"user_tz":420,"elapsed":732,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":["# evaluation result\n","from sklearn.metrics import accuracy_score\n","accuracy_score(y_true, y_pred)"],"execution_count":0,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.504"]},"metadata":{"tags":[]},"execution_count":25}]}]}