notebooks/convolutional_neural_network.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "064d9ad9-113a-412a-880c-7d179549a2df",
   "metadata": {},
   "source": [
    "# Convolutional Neural Network"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5517a4e1-f990-45c2-8385-e99e02630fb7",
   "metadata": {},
   "source": [
    "## Load Dependent Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7f4191ef-7917-477d-a25d-a49e318fce0c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac6dff1d-b8b8-48b9-a312-b52bb5205313",
   "metadata": {},
   "source": [
    "## Device configuration: CPU or GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e60e8012-d126-4815-bdb4-7c12b6ae8ae0",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b495686-5b9c-4baa-87f9-2bfb737097c9",
   "metadata": {},
   "source": [
    "## Select Hyper Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9d7e22a7-f2e3-4fd1-8f47-01c276e3b7f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_epochs = 5\n",
    "num_classes = 10\n",
    "batch_size = 100 # Total 100 batches\n",
    "learning_rate = 0.01"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4ba06ee-cc8a-433c-a143-3326da562e49",
   "metadata": {},
   "source": [
    "## Define Dataset Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "424573d2-a82c-4bb4-bdf2-f3a1e972f19b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import gzip\n",
    "import os\n",
    "class MNISTDataset(torch.utils.data.Dataset):\n",
    "    def __init__(self, root, train=True, transform=None):\n",
    "        # The file name prefix is obtained according to whether it is a training set or not.\n",
    "        self.file_pre = 'train' if train == True else 't10k'\n",
    "        self.transform = transform\n",
    "\n",
    "        # Generate the image and label file path of the corresponding dataset.\n",
    "        self.label_path = os.path.join(root, '%s-labels-idx1-ubyte.gz' % self.file_pre)\n",
    "        self.image_path = os.path.join(root, '%s-images-idx3-ubyte.gz' % self.file_pre)\n",
    "\n",
    "        # Read file data and return pictures and labels.\n",
    "        self.images, self.labels = self.__read_data__(self.image_path, self.label_path)\n",
    "\n",
    "    def __read_data__(self, image_path, label_path):\n",
    "        # Data set reading.\n",
    "        with gzip.open(label_path, 'rb') as lbpath:\n",
    "            labels = np.frombuffer(lbpath.read(), np.uint8, offset=8)\n",
    "        with gzip.open(image_path, 'rb') as imgpath:\n",
    "            images = np.frombuffer(imgpath.read(), np.uint8, offset=16).reshape(len(labels), 28, 28)\n",
    "        return images, labels\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        image, label = self.images[index], int(self.labels[index])\n",
    "        \n",
    "        # If you need to convert to tensor, use tansform.\n",
    "        if self.transform is not None:\n",
    "            image = self.transform(np.array(image))  # Avoid bug: use np.array\n",
    "        return image, label\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "257944ea-96c9-48b7-a2d5-16aaa0276c90",
   "metadata": {},
   "source": [
    "Download Dataset from Online:\n",
    "```python\n",
    "train_dataset = torchvision.datasets.MNIST(root='../data/MNIST', train=True, transform=torchvision.transforms.ToTensor(), download=True)\n",
    "test_dataset = torchvision.datasets.MNIST(root='../data/MNIST', train=False, transform=torchvision.transfroms.ToTensor(), download=True)\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d07cbb6-5e94-480f-a497-14a18604ce64",
   "metadata": {},
   "source": [
    "## Load through Local Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "46c9ab59-d808-4420-aa85-7b30385ce7c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# If datasets have been downloaded already!\n",
    "train_dataset = MNISTDataset('../data/MNIST/', transform=torchvision.transforms.ToTensor())\n",
    "test_dataset = MNISTDataset('../data/MNIST/', train=False, transform=torchvision.transforms.ToTensor())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "95190134-1b2d-4f3b-9861-7bb2b4efd3e0",
   "metadata": {},
   "source": [
    "## Set Data loader (Input Pipeline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "933feab7-39c0-4e7f-a187-b7705c69e024",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46508b86-b8f0-497a-98a5-5f795b034ad8",
   "metadata": {},
   "source": [
    "Calculate the Input Parameters of the First Full Connection Layer:\n",
    "```python\n",
    "def fc_in(image, Conv, Pool):\n",
    "    for i, j in zip(Conv, Pool):\n",
    "        hk = (image[0] - i[0] + 2 * i[2]) / i[1] + 1\n",
    "        wk = (image[1] - i[0] + 2 * i[2]) / i[1] + 1\n",
    "        hp = (hk - j[0] + 2 * j[2]) / j[1] + 1\n",
    "        wp = (wk - j[0] + 2 * j[2]) / j[1] + 1\n",
    "        image = (hp, wp)\n",
    "    return (int(image[0]), int(image[1]))\n",
    "fc_in((28, 28), ((5, 1, 2), (5, 1, 2)), ((2, 2, 0), (2, 2, 0)))\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "609061e6-8df8-4903-a19a-daf6deccd02d",
   "metadata": {},
   "source": [
    "## Define Convolutional Neural Network (Two Convolutional Layers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "588a1284-7b04-4eee-8c4b-1416d0639d9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvNet(torch.nn.Module):\n",
    "    def __init__(self, num_classes):\n",
    "        super(ConvNet, self).__init__()\n",
    "        self.layer1 = torch.nn.Sequential(torch.nn.Conv2d(1, 16, kernel_size=5, stride=1, padding=2),\n",
    "                                          torch.nn.BatchNorm2d(16),\n",
    "                                          torch.nn.ReLU(),\n",
    "                                          torch.nn.MaxPool2d(kernel_size=2, stride=2))\n",
    "        self.layer2 = torch.nn.Sequential(torch.nn.Conv2d(16, 32, kernel_size=5, stride=1, padding=2),\n",
    "                                          torch.nn.BatchNorm2d(32),\n",
    "                                          torch.nn.ReLU(),\n",
    "                                          torch.nn.MaxPool2d(kernel_size=2, stride=2))\n",
    "        self.fc = torch.nn.Linear(7 * 7 * 32, num_classes)\n",
    "     \n",
    "    def forward(self, x):\n",
    "        out = self.layer1(x)\n",
    "        out = self.layer2(out)\n",
    "        # out.size = (batchsize, channels, x, y)\n",
    "        out = out.reshape(out.size(0), -1)\n",
    "        out = self.fc(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b60243fb-5379-4de3-93ce-9acd89c100b3",
   "metadata": {},
   "source": [
    "## Make Model with GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "05b63a23-e7aa-477e-9284-8f20f20174af",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ConvNet(num_classes).to(device)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb62229c-df91-4423-9d15-b876db0eb757",
   "metadata": {},
   "source": [
    "## Loss and Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "a652f5fc-482b-45cf-9c5f-e0273ad88ffc",
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74c87c38-fa93-41d6-9c88-1871b9609316",
   "metadata": {},
   "source": [
    "## Train Model with GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1b2dd35e-7371-4248-a0fd-4c7c8e07437a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [1/5], Step [100/600], Loss: 0.1519\n",
      "Epoch [1/5], Step [200/600], Loss: 0.0671\n",
      "Epoch [1/5], Step [300/600], Loss: 0.1410\n",
      "Epoch [1/5], Step [400/600], Loss: 0.1034\n",
      "Epoch [1/5], Step [500/600], Loss: 0.0701\n",
      "Epoch [1/5], Step [600/600], Loss: 0.0220\n",
      "Epoch [2/5], Step [100/600], Loss: 0.0092\n",
      "Epoch [2/5], Step [200/600], Loss: 0.0312\n",
      "Epoch [2/5], Step [300/600], Loss: 0.1994\n",
      "Epoch [2/5], Step [400/600], Loss: 0.0334\n",
      "Epoch [2/5], Step [500/600], Loss: 0.0097\n",
      "Epoch [2/5], Step [600/600], Loss: 0.0768\n",
      "Epoch [3/5], Step [100/600], Loss: 0.0480\n",
      "Epoch [3/5], Step [200/600], Loss: 0.0338\n",
      "Epoch [3/5], Step [300/600], Loss: 0.0429\n",
      "Epoch [3/5], Step [400/600], Loss: 0.0221\n",
      "Epoch [3/5], Step [500/600], Loss: 0.0179\n",
      "Epoch [3/5], Step [600/600], Loss: 0.0190\n",
      "Epoch [4/5], Step [100/600], Loss: 0.0191\n",
      "Epoch [4/5], Step [200/600], Loss: 0.0046\n",
      "Epoch [4/5], Step [300/600], Loss: 0.0341\n",
      "Epoch [4/5], Step [400/600], Loss: 0.0213\n",
      "Epoch [4/5], Step [500/600], Loss: 0.0509\n",
      "Epoch [4/5], Step [600/600], Loss: 0.0227\n",
      "Epoch [5/5], Step [100/600], Loss: 0.0039\n",
      "Epoch [5/5], Step [200/600], Loss: 0.0025\n",
      "Epoch [5/5], Step [300/600], Loss: 0.0699\n",
      "Epoch [5/5], Step [400/600], Loss: 0.0596\n",
      "Epoch [5/5], Step [500/600], Loss: 0.0031\n",
      "Epoch [5/5], Step [600/600], Loss: 0.0393\n"
     ]
    }
   ],
   "source": [
    "total_step = len(train_loader)\n",
    "for epoch in range(num_epochs):\n",
    "    for i, (images, labels) in enumerate(train_loader):\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        \n",
    "        # Forward pass\n",
    "        outputs = model(images)\n",
    "        loss = criterion(outputs, labels)\n",
    "        \n",
    "        # Backward and optimize\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        if (i+1) % 100 == 0:\n",
    "            print ('Epoch [{}/{}], Step [{}/{}], Loss: {:.4f}'.format(epoch+1, num_epochs, i+1, total_step, loss.item()))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97b0c15f-74f9-4059-aec7-76ed0532b38b",
   "metadata": {},
   "source": [
    "## Test Model with GPU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "9b37a850-58fc-4ede-b404-a3521b91b5f4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Accuracy of the model on the 10000 test images: 98.56 %\n"
     ]
    }
   ],
   "source": [
    "# Do not enable batchnormalization and dropout to ensure that BN and dropout do not change. The pytorch framework will automatically fix BN and dropout without averaging, but use the trained value.\n",
    "model.eval()\n",
    "with torch.no_grad():\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for images, labels in test_loader:\n",
    "        images = images.to(device)\n",
    "        labels = labels.to(device)\n",
    "        outputs = model(images)\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "    print('Test Accuracy of the model on the 10000 test images: {} %'.format(100 * correct / total))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d3f39e76-342f-4d6f-badb-09007004dee8",
   "metadata": {},
   "source": [
    "## Save the Model Checkpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "fe24bbb5-483d-4529-a8d9-bf34bf155fa5",
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model.state_dict(), 'cnn.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bce01c0e-259e-4a15-8c5e-fb714ea014b0",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pytorch",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}