Deep Learning with PyTorch/DL-with-pytorch - 1 [MLP].ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"DL-with-pytorch - 1 [MLP].ipynb","version":"0.3.2","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"metadata":{"id":"bIAYbYajk1w9","colab_type":"text"},"cell_type":"markdown","source":["# Multi Layer Perceptrons\n","- Breast cancer prediction with MLP\n","  - [Breast cancer dataset](https://scikit-learn.org/stable/modules/generated/sklearn.datasets.load_breast_cancer.html#sklearn.datasets.load_breast_cancer)"]},{"metadata":{"id":"GVU5-yp3N89I","colab_type":"code","outputId":"7bd3dd6e-db5b-431b-f695-02353c5e4338","executionInfo":{"status":"ok","timestamp":1545794590415,"user_tz":420,"elapsed":5680,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"colab":{"base_uri":"https://localhost:8080/","height":125}},"cell_type":"code","source":["!pip3 install torch torchvision"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (1.0.0)\n","Requirement already satisfied: torchvision in /usr/local/lib/python3.6/dist-packages (0.2.1)\n","Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from torchvision) (1.11.0)\n","Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from torchvision) (5.3.0)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torchvision) (1.14.6)\n"],"name":"stdout"}]},{"metadata":{"id":"8yy37hEYOEiQ","colab_type":"code","outputId":"8b62db35-c199-4e53-95e5-f40c0dd6b14c","executionInfo":{"status":"ok","timestamp":1545794593243,"user_tz":420,"elapsed":718,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":["import numpy as np\n","import pandas as pd\n","import torch, torchvision\n","import torch.nn as nn\n","import torch.nn.functional as F\n","torch.__version__"],"execution_count":2,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'1.0.0'"]},"metadata":{"tags":[]},"execution_count":2}]},{"metadata":{"id":"ewrw93tt2BfV","colab_type":"text"},"cell_type":"markdown","source":["## 1. Import & process dataset\n"]},{"metadata":{"id":"fCyPxp6QFe59","colab_type":"code","colab":{}},"cell_type":"code","source":["from sklearn.datasets import load_breast_cancer\n","# define dataset\n","class breastCancerDataset(torch.utils.data.Dataset):\n","  def __init__(self):\n","    data = load_breast_cancer()\n","\n","    self.X = data.data\n","    self.y = data.target\n","  \n","  def __getitem__(self, idx):\n","    return self.X[idx], self.y[idx]\n","  \n","  def __len__(self):\n","    return len(self.X)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"sRWTaxYiYweF","colab_type":"code","outputId":"ad1b6590-6533-40ac-8ca0-28585b2e0283","executionInfo":{"status":"ok","timestamp":1545794605379,"user_tz":420,"elapsed":710,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":["# create dataset instance and train-test split\n","dataset = breastCancerDataset()\n","\n","NUM_INSTANCES = len(dataset)\n","TEST_RATIO = 0.4\n","TEST_SIZE = int(NUM_INSTANCES * TEST_RATIO)\n","TRAIN_SIZE = NUM_INSTANCES - TEST_SIZE\n","\n","print(TRAIN_SIZE, TEST_SIZE)"],"execution_count":4,"outputs":[{"output_type":"stream","text":["342 227\n"],"name":"stdout"}]},{"metadata":{"id":"qCJfxbYbFnY3","colab_type":"code","colab":{}},"cell_type":"code","source":["train_data, test_data = torch.utils.data.random_split(dataset, (TRAIN_SIZE, TEST_SIZE))\n","\n","train_loader = torch.utils.data.DataLoader(train_data, batch_size = 128, shuffle = True)\n","test_loader = torch.utils.data.DataLoader(test_data, batch_size = 128, shuffle = False)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"9nznVMdo5edZ","colab_type":"text"},"cell_type":"markdown","source":["## 2. Creating MLP model and training\n","\n","- Create and train MLP with 1 hidden layer\n","\n","![](http://cs231n.github.io/assets/nn1/neural_net.jpeg)"]},{"metadata":{"id":"AQawpMRPI7jm","colab_type":"code","colab":{}},"cell_type":"code","source":["# create MLP with one hidden layer\n","class net(nn.Module):\n","  def __init__(self, input_dim, hidden_dim):\n","    super(net, self).__init__()\n","    self.dense1 = nn.Linear(input_dim, hidden_dim)   # hidden layer\n","    self.relu = nn.ReLU()\n","    self.dense2 = nn.Linear(hidden_dim, 2)           # output layer\n","  \n","  def forward(self, x):\n","    x = self.dense1(x)   \n","    x = self.relu(x)\n","    x = self.dense2(x)\n","    return x"],"execution_count":0,"outputs":[]},{"metadata":{"id":"rP0Gt5E9ajmd","colab_type":"code","colab":{}},"cell_type":"code","source":["# hyperparameters\n","DEVICE = torch.device('cuda')\n","INPUT_DIM = load_breast_cancer().data.shape[-1]\n","HIDDEN_DIM = 5\n","LEARNING_RATE = 1e-3\n","NUM_EPOCHS = 10"],"execution_count":0,"outputs":[]},{"metadata":{"id":"cPBm8qDrSWsi","colab_type":"code","colab":{}},"cell_type":"code","source":["model = net(INPUT_DIM, HIDDEN_DIM).to(DEVICE)\n","criterion = nn.CrossEntropyLoss()   # do not need softmax layer when using CEloss criterion\n","optimizer = torch.optim.RMSprop(model.parameters(), lr = LEARNING_RATE)"],"execution_count":0,"outputs":[]},{"metadata":{"id":"SEBtAPYCFeic","colab_type":"code","outputId":"4280ef9b-4d56-4445-84ca-48fc2d828f67","executionInfo":{"status":"ok","timestamp":1545794653669,"user_tz":420,"elapsed":2454,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"colab":{"base_uri":"https://localhost:8080/","height":193}},"cell_type":"code","source":["# training for NUM_EPOCHS\n","for i in range(NUM_EPOCHS):\n","  temp_loss = []\n","  for (x, y) in train_loader:\n","    x, y = x.float().to(DEVICE), y.to(DEVICE)\n","    outputs = model(x)\n","    loss = criterion(outputs, y)\n","    temp_loss.append(loss.item())\n","    \n","    optimizer.zero_grad()\n","    loss.backward()\n","    optimizer.step()\n","    \n","  print(\"Loss at {}th epoch: {}\".format(i, np.mean(temp_loss)))"],"execution_count":10,"outputs":[{"output_type":"stream","text":["Loss at 0th epoch: 12.068120320638021\n","Loss at 1th epoch: 2.8435821533203125\n","Loss at 2th epoch: 1.4054580926895142\n","Loss at 3th epoch: 1.2583297888437908\n","Loss at 4th epoch: 1.3461804389953613\n","Loss at 5th epoch: 1.047558069229126\n","Loss at 6th epoch: 1.0004488627115886\n","Loss at 7th epoch: 0.8730776309967041\n","Loss at 8th epoch: 0.8132720788319906\n","Loss at 9th epoch: 0.7596675952275594\n"],"name":"stdout"}]},{"metadata":{"id":"qpAJUiHm529m","colab_type":"text"},"cell_type":"markdown","source":["## 3. Evaluation\n","- Evaluate the trained MLP model with ROC AUC score \n","  - Store probability of each instance to a list and compare it with true y label"]},{"metadata":{"id":"txXH3dknFpSx","colab_type":"code","outputId":"96536291-ce13-4162-b60b-915484702ec7","executionInfo":{"status":"ok","timestamp":1545794679211,"user_tz":420,"elapsed":707,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"colab":{"base_uri":"https://localhost:8080/","height":72}},"cell_type":"code","source":["# record probability and true label \n","y_prob, y_true = [], []\n","with torch.no_grad():\n","  for x, y in test_loader:\n","    x, y = x.float().to(DEVICE), y.to(DEVICE)\n","    outputs = F.softmax(model(x))\n","    y_true += list(y.cpu().numpy())                # true label\n","    y_prob += list(outputs.cpu().numpy()[:, -1])   # predicted probability "],"execution_count":11,"outputs":[{"output_type":"stream","text":["/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:5: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n","  \"\"\"\n"],"name":"stderr"}]},{"metadata":{"id":"HV1s3xf5Frkl","colab_type":"code","outputId":"8318627b-018d-4637-cdc5-99d931807156","executionInfo":{"status":"ok","timestamp":1545794680588,"user_tz":420,"elapsed":745,"user":{"displayName":"Buomsoo Kim","photoUrl":"","userId":"18268696804115368229"}},"colab":{"base_uri":"https://localhost:8080/","height":34}},"cell_type":"code","source":["# evaluation result\n","from sklearn.metrics import roc_auc_score\n","roc_auc_score(y_true, y_prob)"],"execution_count":12,"outputs":[{"output_type":"execute_result","data":{"text/plain":["0.5968810601163542"]},"metadata":{"tags":[]},"execution_count":12}]}]}