PeleeNet-PyTorch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0IpEG0n1w8zp"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "from torchvision.utils import save_image\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "import numpy as np\n",
    "import os\n",
    "import glob\n",
    "import PIL\n",
    "from PIL import Image\n",
    "from torch.utils import data as D\n",
    "from torch.utils.data.sampler import SubsetRandomSampler\n",
    "import random\n",
    "import torchsummary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1909,
     "status": "ok",
     "timestamp": 1549779074258,
     "user": {
      "displayName": "PUBG호야",
      "photoUrl": "https://lh3.googleusercontent.com/-995djavMjjs/AAAAAAAAAAI/AAAAAAAAAGI/vwiBSSQLtiQ/s64/photo.jpg",
      "userId": "15725788610401702262"
     },
     "user_tz": -540
    },
    "id": "quWno9W_xJmD",
    "outputId": "6c8d0c21-5289-4ca5-aeba-5dceb3167236"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0.0\n",
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "print(torch.__version__)\n",
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "OLKeDNrgxMJO"
   },
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "validation_ratio = 0.1\n",
    "random_seed = 10\n",
    "initial_lr = 0.1\n",
    "num_epoch = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 68
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 29950,
     "status": "ok",
     "timestamp": 1549779102329,
     "user": {
      "displayName": "PUBG호야",
      "photoUrl": "https://lh3.googleusercontent.com/-995djavMjjs/AAAAAAAAAAI/AAAAAAAAAGI/vwiBSSQLtiQ/s64/photo.jpg",
      "userId": "15725788610401702262"
     },
     "user_tz": -540
    },
    "id": "C-dON3NcxM6c",
    "outputId": "f2df4f46-7318-466b-c179-ab484243ba35"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "transform_train = transforms.Compose([\n",
    "        transforms.Resize(256),\n",
    "        transforms.RandomCrop(224),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))])\n",
    "\n",
    "transform_validation = transforms.Compose([\n",
    "        transforms.Resize(224),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))])\n",
    "\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "        transforms.Resize(224),     \n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2470, 0.2435, 0.2616))])\n",
    "\n",
    "trainset = torchvision.datasets.CIFAR10(\n",
    "    root='./data', train=True, download=True, transform=transform_train)\n",
    "\n",
    "validset = torchvision.datasets.CIFAR10(\n",
    "    root='./data', train=True, download=True, transform=transform_validation)\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(\n",
    "    root='./data', train=False, download=True, transform=transform_test)\n",
    "\n",
    "\n",
    "num_train = len(trainset)\n",
    "indices = list(range(num_train))\n",
    "split = int(np.floor(validation_ratio * num_train))\n",
    "\n",
    "np.random.seed(random_seed)\n",
    "np.random.shuffle(indices)\n",
    "\n",
    "train_idx, valid_idx = indices[split:], indices[:split]\n",
    "train_sampler = SubsetRandomSampler(train_idx)\n",
    "valid_sampler = SubsetRandomSampler(valid_idx)\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "    trainset, batch_size=batch_size, sampler=train_sampler, num_workers=0\n",
    ")\n",
    "\n",
    "valid_loader = torch.utils.data.DataLoader(\n",
    "    validset, batch_size=batch_size, sampler=valid_sampler, num_workers=0\n",
    ")\n",
    "\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "    testset, batch_size=batch_size, shuffle=False, num_workers=0\n",
    ")\n",
    "\n",
    "classes = ('plane', 'car', 'bird', 'cat',\n",
    "           'deer', 'dog', 'frog', 'horse', 'ship', 'truck')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2_Wdkl-TxQBz"
   },
   "outputs": [],
   "source": [
    "class conv_bn_relu(nn.Module):\n",
    "    def __init__(self, nin, nout, kernel_size, stride, padding, bias=False):\n",
    "        super(conv_bn_relu, self).__init__()\n",
    "        self.conv = nn.Conv2d(nin, nout, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias)\n",
    "        self.batch_norm = nn.BatchNorm2d(nout)\n",
    "        self.relu = nn.ReLU(True)\n",
    "\n",
    "    def forward(self, x):\n",
    "        out = self.conv(x)\n",
    "        out = self.batch_norm(out)\n",
    "        out = self.relu(out)\n",
    "\n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2OXj4ClU0t0B"
   },
   "outputs": [],
   "source": [
    "class Transition_layer(nn.Sequential):\n",
    "  def __init__(self, nin, theta=1):    \n",
    "      super(Transition_layer, self).__init__()\n",
    "      \n",
    "      self.add_module('conv_1x1', conv_bn_relu(nin=nin, nout=int(nin*theta), kernel_size=1, stride=1, padding=0, bias=False))\n",
    "      self.add_module('avg_pool_2x2', nn.AvgPool2d(kernel_size=2, stride=2, padding=0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IFlC3oaM0V4q"
   },
   "outputs": [],
   "source": [
    "class StemBlock(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(StemBlock, self).__init__()\n",
    "        \n",
    "        self.conv_3x3_first = conv_bn_relu(nin=3, nout=32, kernel_size=3, stride=2, padding=1, bias=False)\n",
    "        \n",
    "        self.conv_1x1_left = conv_bn_relu(nin=32, nout=16, kernel_size=1, stride=1, padding=0, bias=False)\n",
    "        self.conv_3x3_left = conv_bn_relu(nin=16, nout=32, kernel_size=3, stride=2, padding=1, bias=False)\n",
    "        \n",
    "        self.max_pool_right = nn.MaxPool2d(kernel_size=2, stride=2, padding=0)\n",
    "        \n",
    "        self.conv_1x1_last = conv_bn_relu(nin=64, nout=32, kernel_size=1, stride=1, padding=0, bias=False)\n",
    "\n",
    "    def forward(self, x):\n",
    "        out_first = self.conv_3x3_first(x)\n",
    "        \n",
    "        out_left = self.conv_1x1_left(out_first)\n",
    "        out_left = self.conv_3x3_left(out_left)\n",
    "        \n",
    "        out_right = self.max_pool_right(out_first)\n",
    "        \n",
    "        out_middle = torch.cat((out_left, out_right), 1)\n",
    "        \n",
    "        out_last = self.conv_1x1_last(out_middle)\n",
    "                \n",
    "        return out_last"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aSZQgu02xUKy"
   },
   "outputs": [],
   "source": [
    "class dense_layer(nn.Module):\n",
    "  def __init__(self, nin, growth_rate, drop_rate=0.2):    \n",
    "      super(dense_layer, self).__init__()\n",
    "      \n",
    "      self.dense_left_way = nn.Sequential()\n",
    "      \n",
    "      self.dense_left_way.add_module('conv_1x1', conv_bn_relu(nin=nin, nout=growth_rate*2, kernel_size=1, stride=1, padding=0, bias=False))\n",
    "      self.dense_left_way.add_module('conv_3x3', conv_bn_relu(nin=growth_rate*2, nout=growth_rate//2, kernel_size=3, stride=1, padding=1, bias=False))\n",
    "            \n",
    "      self.dense_right_way = nn.Sequential()\n",
    "      \n",
    "      self.dense_right_way.add_module('conv_1x1', conv_bn_relu(nin=nin, nout=growth_rate*2, kernel_size=1, stride=1, padding=0, bias=False))\n",
    "      self.dense_right_way.add_module('conv_3x3_1', conv_bn_relu(nin=growth_rate*2, nout=growth_rate//2, kernel_size=3, stride=1, padding=1, bias=False))\n",
    "      self.dense_right_way.add_module('conv_3x3 2', conv_bn_relu(nin=growth_rate//2, nout=growth_rate//2, kernel_size=3, stride=1, padding=1, bias=False))\n",
    "      \n",
    "      self.drop_rate = drop_rate\n",
    "      \n",
    "  def forward(self, x):\n",
    "      left_output = self.dense_left_way(x)\n",
    "      right_output = self.dense_right_way(x)\n",
    "\n",
    "      if self.drop_rate > 0:\n",
    "          left_output = F.dropout(left_output, p=self.drop_rate, training=self.training)\n",
    "          right_output = F.dropout(right_output, p=self.drop_rate, training=self.training)\n",
    "          \n",
    "      dense_layer_output = torch.cat((x, left_output, right_output), 1)\n",
    "            \n",
    "      return dense_layer_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zBpPsc18xWiD"
   },
   "outputs": [],
   "source": [
    "class DenseBlock(nn.Sequential):\n",
    "  def __init__(self, nin, num_dense_layers, growth_rate, drop_rate=0.0):\n",
    "      super(DenseBlock, self).__init__()\n",
    "                        \n",
    "      for i in range(num_dense_layers):\n",
    "          nin_dense_layer = nin + growth_rate * i\n",
    "          self.add_module('dense_layer_%d' % i, dense_layer(nin=nin_dense_layer, growth_rate=growth_rate, drop_rate=drop_rate))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mmk5h_QXxYoh"
   },
   "outputs": [],
   "source": [
    "class PeleeNet(nn.Module):\n",
    "    def __init__(self, growth_rate=32, num_dense_layers=[3,4,8,6], theta=1, drop_rate=0.0, num_classes=10):\n",
    "        super(PeleeNet, self).__init__()\n",
    "        \n",
    "        assert len(num_dense_layers) == 4\n",
    "        \n",
    "        self.features = nn.Sequential()\n",
    "        self.features.add_module('StemBlock', StemBlock())\n",
    "        \n",
    "        nin_transition_layer = 32\n",
    "        \n",
    "        for i in range(len(num_dense_layers)):\n",
    "            self.features.add_module('DenseBlock_%d' % (i+1), DenseBlock(nin=nin_transition_layer, num_dense_layers=num_dense_layers[i], growth_rate=growth_rate, drop_rate=0.0))\n",
    "            nin_transition_layer +=  num_dense_layers[i] * growth_rate\n",
    "            \n",
    "            if i == len(num_dense_layers) - 1:\n",
    "                self.features.add_module('Transition_layer_%d' % (i+1), conv_bn_relu(nin=nin_transition_layer, nout=int(nin_transition_layer*theta), kernel_size=1, stride=1, padding=0, bias=False))\n",
    "            else:\n",
    "                self.features.add_module('Transition_layer_%d' % (i+1), Transition_layer(nin=nin_transition_layer, theta=1))\n",
    "        \n",
    "        self.linear = nn.Linear(nin_transition_layer, num_classes)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        stage_output = self.features(x)\n",
    "        \n",
    "        global_avg_pool_output = F.adaptive_avg_pool2d(stage_output, (1, 1))  \n",
    "        global_avg_pool_output_flat = global_avg_pool_output.view(global_avg_pool_output.size(0), -1)\n",
    "                \n",
    "        output = self.linear(global_avg_pool_output_flat)\n",
    "        \n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rpslu5POxbjD"
   },
   "outputs": [],
   "source": [
    "net = PeleeNet()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 12189
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 35784,
     "status": "ok",
     "timestamp": 1549779108241,
     "user": {
      "displayName": "PUBG호야",
      "photoUrl": "https://lh3.googleusercontent.com/-995djavMjjs/AAAAAAAAAAI/AAAAAAAAAGI/vwiBSSQLtiQ/s64/photo.jpg",
      "userId": "15725788610401702262"
     },
     "user_tz": -540
    },
    "id": "BxJpVdKDDrA4",
    "outputId": "652bb643-0b45-4e40-8d64-da6b0278b73a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PeleeNet(\n",
       "  (features): Sequential(\n",
       "    (StemBlock): StemBlock(\n",
       "      (conv_3x3_first): conv_bn_relu(\n",
       "        (conv): Conv2d(3, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        (batch_norm): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (conv_1x1_left): conv_bn_relu(\n",
       "        (conv): Conv2d(32, 16, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (conv_3x3_left): conv_bn_relu(\n",
       "        (conv): Conv2d(16, 32, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        (batch_norm): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (max_pool_right): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "      (conv_1x1_last): conv_bn_relu(\n",
       "        (conv): Conv2d(64, 32, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (batch_norm): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "    )\n",
       "    (DenseBlock_1): DenseBlock(\n",
       "      (dense_layer_0): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(32, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (dense_layer_1): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (dense_layer_2): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(96, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(96, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (Transition_layer_1): Transition_layer(\n",
       "      (conv_1x1): conv_bn_relu(\n",
       "        (conv): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (batch_norm): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (avg_pool_2x2): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
       "    )\n",
       "    (DenseBlock_2): DenseBlock(\n",
       "      (dense_layer_0): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(128, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(128, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (dense_layer_1): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(160, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(160, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (dense_layer_2): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(192, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(192, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (dense_layer_3): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(224, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(224, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (Transition_layer_2): Transition_layer(\n",
       "      (conv_1x1): conv_bn_relu(\n",
       "        (conv): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (batch_norm): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (avg_pool_2x2): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
       "    )\n",
       "    (DenseBlock_3): DenseBlock(\n",
       "      (dense_layer_0): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (dense_layer_1): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(288, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(288, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (dense_layer_2): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(320, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(320, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (dense_layer_3): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(352, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(352, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (dense_layer_4): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(384, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(384, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (dense_layer_5): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(416, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(416, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (dense_layer_6): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(448, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(448, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (dense_layer_7): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(480, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(480, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (Transition_layer_3): Transition_layer(\n",
       "      (conv_1x1): conv_bn_relu(\n",
       "        (conv): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "        (batch_norm): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "      )\n",
       "      (avg_pool_2x2): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
       "    )\n",
       "    (DenseBlock_4): DenseBlock(\n",
       "      (dense_layer_0): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(512, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(512, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (dense_layer_1): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(544, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(544, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (dense_layer_2): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(576, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(576, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (dense_layer_3): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(608, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(608, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (dense_layer_4): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(640, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(640, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "      (dense_layer_5): dense_layer(\n",
       "        (dense_left_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(672, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "        (dense_right_way): Sequential(\n",
       "          (conv_1x1): conv_bn_relu(\n",
       "            (conv): Conv2d(672, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3_1): conv_bn_relu(\n",
       "            (conv): Conv2d(64, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "          (conv_3x3 2): conv_bn_relu(\n",
       "            (conv): Conv2d(16, 16, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "            (batch_norm): BatchNorm2d(16, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "            (relu): ReLU(inplace)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (Transition_layer_4): conv_bn_relu(\n",
       "      (conv): Conv2d(704, 704, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
       "      (batch_norm): BatchNorm2d(704, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      (relu): ReLU(inplace)\n",
       "    )\n",
       "  )\n",
       "  (linear): Linear(in_features=704, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 8381
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 36294,
     "status": "ok",
     "timestamp": 1549779108764,
     "user": {
      "displayName": "PUBG호야",
      "photoUrl": "https://lh3.googleusercontent.com/-995djavMjjs/AAAAAAAAAAI/AAAAAAAAAGI/vwiBSSQLtiQ/s64/photo.jpg",
      "userId": "15725788610401702262"
     },
     "user_tz": -540
    },
    "id": "3qYqt69xDfrP",
    "outputId": "1400db13-0547-4a30-fdad-e16fc5d47d97"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1         [-1, 32, 112, 112]             864\n",
      "       BatchNorm2d-2         [-1, 32, 112, 112]              64\n",
      "              ReLU-3         [-1, 32, 112, 112]               0\n",
      "      conv_bn_relu-4         [-1, 32, 112, 112]               0\n",
      "            Conv2d-5         [-1, 16, 112, 112]             512\n",
      "       BatchNorm2d-6         [-1, 16, 112, 112]              32\n",
      "              ReLU-7         [-1, 16, 112, 112]               0\n",
      "      conv_bn_relu-8         [-1, 16, 112, 112]               0\n",
      "            Conv2d-9           [-1, 32, 56, 56]           4,608\n",
      "      BatchNorm2d-10           [-1, 32, 56, 56]              64\n",
      "             ReLU-11           [-1, 32, 56, 56]               0\n",
      "     conv_bn_relu-12           [-1, 32, 56, 56]               0\n",
      "        MaxPool2d-13           [-1, 32, 56, 56]               0\n",
      "           Conv2d-14           [-1, 32, 56, 56]           2,048\n",
      "      BatchNorm2d-15           [-1, 32, 56, 56]              64\n",
      "             ReLU-16           [-1, 32, 56, 56]               0\n",
      "     conv_bn_relu-17           [-1, 32, 56, 56]               0\n",
      "        StemBlock-18           [-1, 32, 56, 56]               0\n",
      "           Conv2d-19           [-1, 64, 56, 56]           2,048\n",
      "      BatchNorm2d-20           [-1, 64, 56, 56]             128\n",
      "             ReLU-21           [-1, 64, 56, 56]               0\n",
      "     conv_bn_relu-22           [-1, 64, 56, 56]               0\n",
      "           Conv2d-23           [-1, 16, 56, 56]           9,216\n",
      "      BatchNorm2d-24           [-1, 16, 56, 56]              32\n",
      "             ReLU-25           [-1, 16, 56, 56]               0\n",
      "     conv_bn_relu-26           [-1, 16, 56, 56]               0\n",
      "           Conv2d-27           [-1, 64, 56, 56]           2,048\n",
      "      BatchNorm2d-28           [-1, 64, 56, 56]             128\n",
      "             ReLU-29           [-1, 64, 56, 56]               0\n",
      "     conv_bn_relu-30           [-1, 64, 56, 56]               0\n",
      "           Conv2d-31           [-1, 16, 56, 56]           9,216\n",
      "      BatchNorm2d-32           [-1, 16, 56, 56]              32\n",
      "             ReLU-33           [-1, 16, 56, 56]               0\n",
      "     conv_bn_relu-34           [-1, 16, 56, 56]               0\n",
      "           Conv2d-35           [-1, 16, 56, 56]           2,304\n",
      "      BatchNorm2d-36           [-1, 16, 56, 56]              32\n",
      "             ReLU-37           [-1, 16, 56, 56]               0\n",
      "     conv_bn_relu-38           [-1, 16, 56, 56]               0\n",
      "      dense_layer-39           [-1, 64, 56, 56]               0\n",
      "           Conv2d-40           [-1, 64, 56, 56]           4,096\n",
      "      BatchNorm2d-41           [-1, 64, 56, 56]             128\n",
      "             ReLU-42           [-1, 64, 56, 56]               0\n",
      "     conv_bn_relu-43           [-1, 64, 56, 56]               0\n",
      "           Conv2d-44           [-1, 16, 56, 56]           9,216\n",
      "      BatchNorm2d-45           [-1, 16, 56, 56]              32\n",
      "             ReLU-46           [-1, 16, 56, 56]               0\n",
      "     conv_bn_relu-47           [-1, 16, 56, 56]               0\n",
      "           Conv2d-48           [-1, 64, 56, 56]           4,096\n",
      "      BatchNorm2d-49           [-1, 64, 56, 56]             128\n",
      "             ReLU-50           [-1, 64, 56, 56]               0\n",
      "     conv_bn_relu-51           [-1, 64, 56, 56]               0\n",
      "           Conv2d-52           [-1, 16, 56, 56]           9,216\n",
      "      BatchNorm2d-53           [-1, 16, 56, 56]              32\n",
      "             ReLU-54           [-1, 16, 56, 56]               0\n",
      "     conv_bn_relu-55           [-1, 16, 56, 56]               0\n",
      "           Conv2d-56           [-1, 16, 56, 56]           2,304\n",
      "      BatchNorm2d-57           [-1, 16, 56, 56]              32\n",
      "             ReLU-58           [-1, 16, 56, 56]               0\n",
      "     conv_bn_relu-59           [-1, 16, 56, 56]               0\n",
      "      dense_layer-60           [-1, 96, 56, 56]               0\n",
      "           Conv2d-61           [-1, 64, 56, 56]           6,144\n",
      "      BatchNorm2d-62           [-1, 64, 56, 56]             128\n",
      "             ReLU-63           [-1, 64, 56, 56]               0\n",
      "     conv_bn_relu-64           [-1, 64, 56, 56]               0\n",
      "           Conv2d-65           [-1, 16, 56, 56]           9,216\n",
      "      BatchNorm2d-66           [-1, 16, 56, 56]              32\n",
      "             ReLU-67           [-1, 16, 56, 56]               0\n",
      "     conv_bn_relu-68           [-1, 16, 56, 56]               0\n",
      "           Conv2d-69           [-1, 64, 56, 56]           6,144\n",
      "      BatchNorm2d-70           [-1, 64, 56, 56]             128\n",
      "             ReLU-71           [-1, 64, 56, 56]               0\n",
      "     conv_bn_relu-72           [-1, 64, 56, 56]               0\n",
      "           Conv2d-73           [-1, 16, 56, 56]           9,216\n",
      "      BatchNorm2d-74           [-1, 16, 56, 56]              32\n",
      "             ReLU-75           [-1, 16, 56, 56]               0\n",
      "     conv_bn_relu-76           [-1, 16, 56, 56]               0\n",
      "           Conv2d-77           [-1, 16, 56, 56]           2,304\n",
      "      BatchNorm2d-78           [-1, 16, 56, 56]              32\n",
      "             ReLU-79           [-1, 16, 56, 56]               0\n",
      "     conv_bn_relu-80           [-1, 16, 56, 56]               0\n",
      "      dense_layer-81          [-1, 128, 56, 56]               0\n",
      "           Conv2d-82          [-1, 128, 56, 56]          16,384\n",
      "      BatchNorm2d-83          [-1, 128, 56, 56]             256\n",
      "             ReLU-84          [-1, 128, 56, 56]               0\n",
      "     conv_bn_relu-85          [-1, 128, 56, 56]               0\n",
      "        AvgPool2d-86          [-1, 128, 28, 28]               0\n",
      "           Conv2d-87           [-1, 64, 28, 28]           8,192\n",
      "      BatchNorm2d-88           [-1, 64, 28, 28]             128\n",
      "             ReLU-89           [-1, 64, 28, 28]               0\n",
      "     conv_bn_relu-90           [-1, 64, 28, 28]               0\n",
      "           Conv2d-91           [-1, 16, 28, 28]           9,216\n",
      "      BatchNorm2d-92           [-1, 16, 28, 28]              32\n",
      "             ReLU-93           [-1, 16, 28, 28]               0\n",
      "     conv_bn_relu-94           [-1, 16, 28, 28]               0\n",
      "           Conv2d-95           [-1, 64, 28, 28]           8,192\n",
      "      BatchNorm2d-96           [-1, 64, 28, 28]             128\n",
      "             ReLU-97           [-1, 64, 28, 28]               0\n",
      "     conv_bn_relu-98           [-1, 64, 28, 28]               0\n",
      "           Conv2d-99           [-1, 16, 28, 28]           9,216\n",
      "     BatchNorm2d-100           [-1, 16, 28, 28]              32\n",
      "            ReLU-101           [-1, 16, 28, 28]               0\n",
      "    conv_bn_relu-102           [-1, 16, 28, 28]               0\n",
      "          Conv2d-103           [-1, 16, 28, 28]           2,304\n",
      "     BatchNorm2d-104           [-1, 16, 28, 28]              32\n",
      "            ReLU-105           [-1, 16, 28, 28]               0\n",
      "    conv_bn_relu-106           [-1, 16, 28, 28]               0\n",
      "     dense_layer-107          [-1, 160, 28, 28]               0\n",
      "          Conv2d-108           [-1, 64, 28, 28]          10,240\n",
      "     BatchNorm2d-109           [-1, 64, 28, 28]             128\n",
      "            ReLU-110           [-1, 64, 28, 28]               0\n",
      "    conv_bn_relu-111           [-1, 64, 28, 28]               0\n",
      "          Conv2d-112           [-1, 16, 28, 28]           9,216\n",
      "     BatchNorm2d-113           [-1, 16, 28, 28]              32\n",
      "            ReLU-114           [-1, 16, 28, 28]               0\n",
      "    conv_bn_relu-115           [-1, 16, 28, 28]               0\n",
      "          Conv2d-116           [-1, 64, 28, 28]          10,240\n",
      "     BatchNorm2d-117           [-1, 64, 28, 28]             128\n",
      "            ReLU-118           [-1, 64, 28, 28]               0\n",
      "    conv_bn_relu-119           [-1, 64, 28, 28]               0\n",
      "          Conv2d-120           [-1, 16, 28, 28]           9,216\n",
      "     BatchNorm2d-121           [-1, 16, 28, 28]              32\n",
      "            ReLU-122           [-1, 16, 28, 28]               0\n",
      "    conv_bn_relu-123           [-1, 16, 28, 28]               0\n",
      "          Conv2d-124           [-1, 16, 28, 28]           2,304\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     BatchNorm2d-125           [-1, 16, 28, 28]              32\n",
      "            ReLU-126           [-1, 16, 28, 28]               0\n",
      "    conv_bn_relu-127           [-1, 16, 28, 28]               0\n",
      "     dense_layer-128          [-1, 192, 28, 28]               0\n",
      "          Conv2d-129           [-1, 64, 28, 28]          12,288\n",
      "     BatchNorm2d-130           [-1, 64, 28, 28]             128\n",
      "            ReLU-131           [-1, 64, 28, 28]               0\n",
      "    conv_bn_relu-132           [-1, 64, 28, 28]               0\n",
      "          Conv2d-133           [-1, 16, 28, 28]           9,216\n",
      "     BatchNorm2d-134           [-1, 16, 28, 28]              32\n",
      "            ReLU-135           [-1, 16, 28, 28]               0\n",
      "    conv_bn_relu-136           [-1, 16, 28, 28]               0\n",
      "          Conv2d-137           [-1, 64, 28, 28]          12,288\n",
      "     BatchNorm2d-138           [-1, 64, 28, 28]             128\n",
      "            ReLU-139           [-1, 64, 28, 28]               0\n",
      "    conv_bn_relu-140           [-1, 64, 28, 28]               0\n",
      "          Conv2d-141           [-1, 16, 28, 28]           9,216\n",
      "     BatchNorm2d-142           [-1, 16, 28, 28]              32\n",
      "            ReLU-143           [-1, 16, 28, 28]               0\n",
      "    conv_bn_relu-144           [-1, 16, 28, 28]               0\n",
      "          Conv2d-145           [-1, 16, 28, 28]           2,304\n",
      "     BatchNorm2d-146           [-1, 16, 28, 28]              32\n",
      "            ReLU-147           [-1, 16, 28, 28]               0\n",
      "    conv_bn_relu-148           [-1, 16, 28, 28]               0\n",
      "     dense_layer-149          [-1, 224, 28, 28]               0\n",
      "          Conv2d-150           [-1, 64, 28, 28]          14,336\n",
      "     BatchNorm2d-151           [-1, 64, 28, 28]             128\n",
      "            ReLU-152           [-1, 64, 28, 28]               0\n",
      "    conv_bn_relu-153           [-1, 64, 28, 28]               0\n",
      "          Conv2d-154           [-1, 16, 28, 28]           9,216\n",
      "     BatchNorm2d-155           [-1, 16, 28, 28]              32\n",
      "            ReLU-156           [-1, 16, 28, 28]               0\n",
      "    conv_bn_relu-157           [-1, 16, 28, 28]               0\n",
      "          Conv2d-158           [-1, 64, 28, 28]          14,336\n",
      "     BatchNorm2d-159           [-1, 64, 28, 28]             128\n",
      "            ReLU-160           [-1, 64, 28, 28]               0\n",
      "    conv_bn_relu-161           [-1, 64, 28, 28]               0\n",
      "          Conv2d-162           [-1, 16, 28, 28]           9,216\n",
      "     BatchNorm2d-163           [-1, 16, 28, 28]              32\n",
      "            ReLU-164           [-1, 16, 28, 28]               0\n",
      "    conv_bn_relu-165           [-1, 16, 28, 28]               0\n",
      "          Conv2d-166           [-1, 16, 28, 28]           2,304\n",
      "     BatchNorm2d-167           [-1, 16, 28, 28]              32\n",
      "            ReLU-168           [-1, 16, 28, 28]               0\n",
      "    conv_bn_relu-169           [-1, 16, 28, 28]               0\n",
      "     dense_layer-170          [-1, 256, 28, 28]               0\n",
      "          Conv2d-171          [-1, 256, 28, 28]          65,536\n",
      "     BatchNorm2d-172          [-1, 256, 28, 28]             512\n",
      "            ReLU-173          [-1, 256, 28, 28]               0\n",
      "    conv_bn_relu-174          [-1, 256, 28, 28]               0\n",
      "       AvgPool2d-175          [-1, 256, 14, 14]               0\n",
      "          Conv2d-176           [-1, 64, 14, 14]          16,384\n",
      "     BatchNorm2d-177           [-1, 64, 14, 14]             128\n",
      "            ReLU-178           [-1, 64, 14, 14]               0\n",
      "    conv_bn_relu-179           [-1, 64, 14, 14]               0\n",
      "          Conv2d-180           [-1, 16, 14, 14]           9,216\n",
      "     BatchNorm2d-181           [-1, 16, 14, 14]              32\n",
      "            ReLU-182           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-183           [-1, 16, 14, 14]               0\n",
      "          Conv2d-184           [-1, 64, 14, 14]          16,384\n",
      "     BatchNorm2d-185           [-1, 64, 14, 14]             128\n",
      "            ReLU-186           [-1, 64, 14, 14]               0\n",
      "    conv_bn_relu-187           [-1, 64, 14, 14]               0\n",
      "          Conv2d-188           [-1, 16, 14, 14]           9,216\n",
      "     BatchNorm2d-189           [-1, 16, 14, 14]              32\n",
      "            ReLU-190           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-191           [-1, 16, 14, 14]               0\n",
      "          Conv2d-192           [-1, 16, 14, 14]           2,304\n",
      "     BatchNorm2d-193           [-1, 16, 14, 14]              32\n",
      "            ReLU-194           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-195           [-1, 16, 14, 14]               0\n",
      "     dense_layer-196          [-1, 288, 14, 14]               0\n",
      "          Conv2d-197           [-1, 64, 14, 14]          18,432\n",
      "     BatchNorm2d-198           [-1, 64, 14, 14]             128\n",
      "            ReLU-199           [-1, 64, 14, 14]               0\n",
      "    conv_bn_relu-200           [-1, 64, 14, 14]               0\n",
      "          Conv2d-201           [-1, 16, 14, 14]           9,216\n",
      "     BatchNorm2d-202           [-1, 16, 14, 14]              32\n",
      "            ReLU-203           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-204           [-1, 16, 14, 14]               0\n",
      "          Conv2d-205           [-1, 64, 14, 14]          18,432\n",
      "     BatchNorm2d-206           [-1, 64, 14, 14]             128\n",
      "            ReLU-207           [-1, 64, 14, 14]               0\n",
      "    conv_bn_relu-208           [-1, 64, 14, 14]               0\n",
      "          Conv2d-209           [-1, 16, 14, 14]           9,216\n",
      "     BatchNorm2d-210           [-1, 16, 14, 14]              32\n",
      "            ReLU-211           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-212           [-1, 16, 14, 14]               0\n",
      "          Conv2d-213           [-1, 16, 14, 14]           2,304\n",
      "     BatchNorm2d-214           [-1, 16, 14, 14]              32\n",
      "            ReLU-215           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-216           [-1, 16, 14, 14]               0\n",
      "     dense_layer-217          [-1, 320, 14, 14]               0\n",
      "          Conv2d-218           [-1, 64, 14, 14]          20,480\n",
      "     BatchNorm2d-219           [-1, 64, 14, 14]             128\n",
      "            ReLU-220           [-1, 64, 14, 14]               0\n",
      "    conv_bn_relu-221           [-1, 64, 14, 14]               0\n",
      "          Conv2d-222           [-1, 16, 14, 14]           9,216\n",
      "     BatchNorm2d-223           [-1, 16, 14, 14]              32\n",
      "            ReLU-224           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-225           [-1, 16, 14, 14]               0\n",
      "          Conv2d-226           [-1, 64, 14, 14]          20,480\n",
      "     BatchNorm2d-227           [-1, 64, 14, 14]             128\n",
      "            ReLU-228           [-1, 64, 14, 14]               0\n",
      "    conv_bn_relu-229           [-1, 64, 14, 14]               0\n",
      "          Conv2d-230           [-1, 16, 14, 14]           9,216\n",
      "     BatchNorm2d-231           [-1, 16, 14, 14]              32\n",
      "            ReLU-232           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-233           [-1, 16, 14, 14]               0\n",
      "          Conv2d-234           [-1, 16, 14, 14]           2,304\n",
      "     BatchNorm2d-235           [-1, 16, 14, 14]              32\n",
      "            ReLU-236           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-237           [-1, 16, 14, 14]               0\n",
      "     dense_layer-238          [-1, 352, 14, 14]               0\n",
      "          Conv2d-239           [-1, 64, 14, 14]          22,528\n",
      "     BatchNorm2d-240           [-1, 64, 14, 14]             128\n",
      "            ReLU-241           [-1, 64, 14, 14]               0\n",
      "    conv_bn_relu-242           [-1, 64, 14, 14]               0\n",
      "          Conv2d-243           [-1, 16, 14, 14]           9,216\n",
      "     BatchNorm2d-244           [-1, 16, 14, 14]              32\n",
      "            ReLU-245           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-246           [-1, 16, 14, 14]               0\n",
      "          Conv2d-247           [-1, 64, 14, 14]          22,528\n",
      "     BatchNorm2d-248           [-1, 64, 14, 14]             128\n",
      "            ReLU-249           [-1, 64, 14, 14]               0\n",
      "    conv_bn_relu-250           [-1, 64, 14, 14]               0\n",
      "          Conv2d-251           [-1, 16, 14, 14]           9,216\n",
      "     BatchNorm2d-252           [-1, 16, 14, 14]              32\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            ReLU-253           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-254           [-1, 16, 14, 14]               0\n",
      "          Conv2d-255           [-1, 16, 14, 14]           2,304\n",
      "     BatchNorm2d-256           [-1, 16, 14, 14]              32\n",
      "            ReLU-257           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-258           [-1, 16, 14, 14]               0\n",
      "     dense_layer-259          [-1, 384, 14, 14]               0\n",
      "          Conv2d-260           [-1, 64, 14, 14]          24,576\n",
      "     BatchNorm2d-261           [-1, 64, 14, 14]             128\n",
      "            ReLU-262           [-1, 64, 14, 14]               0\n",
      "    conv_bn_relu-263           [-1, 64, 14, 14]               0\n",
      "          Conv2d-264           [-1, 16, 14, 14]           9,216\n",
      "     BatchNorm2d-265           [-1, 16, 14, 14]              32\n",
      "            ReLU-266           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-267           [-1, 16, 14, 14]               0\n",
      "          Conv2d-268           [-1, 64, 14, 14]          24,576\n",
      "     BatchNorm2d-269           [-1, 64, 14, 14]             128\n",
      "            ReLU-270           [-1, 64, 14, 14]               0\n",
      "    conv_bn_relu-271           [-1, 64, 14, 14]               0\n",
      "          Conv2d-272           [-1, 16, 14, 14]           9,216\n",
      "     BatchNorm2d-273           [-1, 16, 14, 14]              32\n",
      "            ReLU-274           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-275           [-1, 16, 14, 14]               0\n",
      "          Conv2d-276           [-1, 16, 14, 14]           2,304\n",
      "     BatchNorm2d-277           [-1, 16, 14, 14]              32\n",
      "            ReLU-278           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-279           [-1, 16, 14, 14]               0\n",
      "     dense_layer-280          [-1, 416, 14, 14]               0\n",
      "          Conv2d-281           [-1, 64, 14, 14]          26,624\n",
      "     BatchNorm2d-282           [-1, 64, 14, 14]             128\n",
      "            ReLU-283           [-1, 64, 14, 14]               0\n",
      "    conv_bn_relu-284           [-1, 64, 14, 14]               0\n",
      "          Conv2d-285           [-1, 16, 14, 14]           9,216\n",
      "     BatchNorm2d-286           [-1, 16, 14, 14]              32\n",
      "            ReLU-287           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-288           [-1, 16, 14, 14]               0\n",
      "          Conv2d-289           [-1, 64, 14, 14]          26,624\n",
      "     BatchNorm2d-290           [-1, 64, 14, 14]             128\n",
      "            ReLU-291           [-1, 64, 14, 14]               0\n",
      "    conv_bn_relu-292           [-1, 64, 14, 14]               0\n",
      "          Conv2d-293           [-1, 16, 14, 14]           9,216\n",
      "     BatchNorm2d-294           [-1, 16, 14, 14]              32\n",
      "            ReLU-295           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-296           [-1, 16, 14, 14]               0\n",
      "          Conv2d-297           [-1, 16, 14, 14]           2,304\n",
      "     BatchNorm2d-298           [-1, 16, 14, 14]              32\n",
      "            ReLU-299           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-300           [-1, 16, 14, 14]               0\n",
      "     dense_layer-301          [-1, 448, 14, 14]               0\n",
      "          Conv2d-302           [-1, 64, 14, 14]          28,672\n",
      "     BatchNorm2d-303           [-1, 64, 14, 14]             128\n",
      "            ReLU-304           [-1, 64, 14, 14]               0\n",
      "    conv_bn_relu-305           [-1, 64, 14, 14]               0\n",
      "          Conv2d-306           [-1, 16, 14, 14]           9,216\n",
      "     BatchNorm2d-307           [-1, 16, 14, 14]              32\n",
      "            ReLU-308           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-309           [-1, 16, 14, 14]               0\n",
      "          Conv2d-310           [-1, 64, 14, 14]          28,672\n",
      "     BatchNorm2d-311           [-1, 64, 14, 14]             128\n",
      "            ReLU-312           [-1, 64, 14, 14]               0\n",
      "    conv_bn_relu-313           [-1, 64, 14, 14]               0\n",
      "          Conv2d-314           [-1, 16, 14, 14]           9,216\n",
      "     BatchNorm2d-315           [-1, 16, 14, 14]              32\n",
      "            ReLU-316           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-317           [-1, 16, 14, 14]               0\n",
      "          Conv2d-318           [-1, 16, 14, 14]           2,304\n",
      "     BatchNorm2d-319           [-1, 16, 14, 14]              32\n",
      "            ReLU-320           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-321           [-1, 16, 14, 14]               0\n",
      "     dense_layer-322          [-1, 480, 14, 14]               0\n",
      "          Conv2d-323           [-1, 64, 14, 14]          30,720\n",
      "     BatchNorm2d-324           [-1, 64, 14, 14]             128\n",
      "            ReLU-325           [-1, 64, 14, 14]               0\n",
      "    conv_bn_relu-326           [-1, 64, 14, 14]               0\n",
      "          Conv2d-327           [-1, 16, 14, 14]           9,216\n",
      "     BatchNorm2d-328           [-1, 16, 14, 14]              32\n",
      "            ReLU-329           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-330           [-1, 16, 14, 14]               0\n",
      "          Conv2d-331           [-1, 64, 14, 14]          30,720\n",
      "     BatchNorm2d-332           [-1, 64, 14, 14]             128\n",
      "            ReLU-333           [-1, 64, 14, 14]               0\n",
      "    conv_bn_relu-334           [-1, 64, 14, 14]               0\n",
      "          Conv2d-335           [-1, 16, 14, 14]           9,216\n",
      "     BatchNorm2d-336           [-1, 16, 14, 14]              32\n",
      "            ReLU-337           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-338           [-1, 16, 14, 14]               0\n",
      "          Conv2d-339           [-1, 16, 14, 14]           2,304\n",
      "     BatchNorm2d-340           [-1, 16, 14, 14]              32\n",
      "            ReLU-341           [-1, 16, 14, 14]               0\n",
      "    conv_bn_relu-342           [-1, 16, 14, 14]               0\n",
      "     dense_layer-343          [-1, 512, 14, 14]               0\n",
      "          Conv2d-344          [-1, 512, 14, 14]         262,144\n",
      "     BatchNorm2d-345          [-1, 512, 14, 14]           1,024\n",
      "            ReLU-346          [-1, 512, 14, 14]               0\n",
      "    conv_bn_relu-347          [-1, 512, 14, 14]               0\n",
      "       AvgPool2d-348            [-1, 512, 7, 7]               0\n",
      "          Conv2d-349             [-1, 64, 7, 7]          32,768\n",
      "     BatchNorm2d-350             [-1, 64, 7, 7]             128\n",
      "            ReLU-351             [-1, 64, 7, 7]               0\n",
      "    conv_bn_relu-352             [-1, 64, 7, 7]               0\n",
      "          Conv2d-353             [-1, 16, 7, 7]           9,216\n",
      "     BatchNorm2d-354             [-1, 16, 7, 7]              32\n",
      "            ReLU-355             [-1, 16, 7, 7]               0\n",
      "    conv_bn_relu-356             [-1, 16, 7, 7]               0\n",
      "          Conv2d-357             [-1, 64, 7, 7]          32,768\n",
      "     BatchNorm2d-358             [-1, 64, 7, 7]             128\n",
      "            ReLU-359             [-1, 64, 7, 7]               0\n",
      "    conv_bn_relu-360             [-1, 64, 7, 7]               0\n",
      "          Conv2d-361             [-1, 16, 7, 7]           9,216\n",
      "     BatchNorm2d-362             [-1, 16, 7, 7]              32\n",
      "            ReLU-363             [-1, 16, 7, 7]               0\n",
      "    conv_bn_relu-364             [-1, 16, 7, 7]               0\n",
      "          Conv2d-365             [-1, 16, 7, 7]           2,304\n",
      "     BatchNorm2d-366             [-1, 16, 7, 7]              32\n",
      "            ReLU-367             [-1, 16, 7, 7]               0\n",
      "    conv_bn_relu-368             [-1, 16, 7, 7]               0\n",
      "     dense_layer-369            [-1, 544, 7, 7]               0\n",
      "          Conv2d-370             [-1, 64, 7, 7]          34,816\n",
      "     BatchNorm2d-371             [-1, 64, 7, 7]             128\n",
      "            ReLU-372             [-1, 64, 7, 7]               0\n",
      "    conv_bn_relu-373             [-1, 64, 7, 7]               0\n",
      "          Conv2d-374             [-1, 16, 7, 7]           9,216\n",
      "     BatchNorm2d-375             [-1, 16, 7, 7]              32\n",
      "            ReLU-376             [-1, 16, 7, 7]               0\n",
      "    conv_bn_relu-377             [-1, 16, 7, 7]               0\n",
      "          Conv2d-378             [-1, 64, 7, 7]          34,816\n",
      "     BatchNorm2d-379             [-1, 64, 7, 7]             128\n",
      "            ReLU-380             [-1, 64, 7, 7]               0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "    conv_bn_relu-381             [-1, 64, 7, 7]               0\n",
      "          Conv2d-382             [-1, 16, 7, 7]           9,216\n",
      "     BatchNorm2d-383             [-1, 16, 7, 7]              32\n",
      "            ReLU-384             [-1, 16, 7, 7]               0\n",
      "    conv_bn_relu-385             [-1, 16, 7, 7]               0\n",
      "          Conv2d-386             [-1, 16, 7, 7]           2,304\n",
      "     BatchNorm2d-387             [-1, 16, 7, 7]              32\n",
      "            ReLU-388             [-1, 16, 7, 7]               0\n",
      "    conv_bn_relu-389             [-1, 16, 7, 7]               0\n",
      "     dense_layer-390            [-1, 576, 7, 7]               0\n",
      "          Conv2d-391             [-1, 64, 7, 7]          36,864\n",
      "     BatchNorm2d-392             [-1, 64, 7, 7]             128\n",
      "            ReLU-393             [-1, 64, 7, 7]               0\n",
      "    conv_bn_relu-394             [-1, 64, 7, 7]               0\n",
      "          Conv2d-395             [-1, 16, 7, 7]           9,216\n",
      "     BatchNorm2d-396             [-1, 16, 7, 7]              32\n",
      "            ReLU-397             [-1, 16, 7, 7]               0\n",
      "    conv_bn_relu-398             [-1, 16, 7, 7]               0\n",
      "          Conv2d-399             [-1, 64, 7, 7]          36,864\n",
      "     BatchNorm2d-400             [-1, 64, 7, 7]             128\n",
      "            ReLU-401             [-1, 64, 7, 7]               0\n",
      "    conv_bn_relu-402             [-1, 64, 7, 7]               0\n",
      "          Conv2d-403             [-1, 16, 7, 7]           9,216\n",
      "     BatchNorm2d-404             [-1, 16, 7, 7]              32\n",
      "            ReLU-405             [-1, 16, 7, 7]               0\n",
      "    conv_bn_relu-406             [-1, 16, 7, 7]               0\n",
      "          Conv2d-407             [-1, 16, 7, 7]           2,304\n",
      "     BatchNorm2d-408             [-1, 16, 7, 7]              32\n",
      "            ReLU-409             [-1, 16, 7, 7]               0\n",
      "    conv_bn_relu-410             [-1, 16, 7, 7]               0\n",
      "     dense_layer-411            [-1, 608, 7, 7]               0\n",
      "          Conv2d-412             [-1, 64, 7, 7]          38,912\n",
      "     BatchNorm2d-413             [-1, 64, 7, 7]             128\n",
      "            ReLU-414             [-1, 64, 7, 7]               0\n",
      "    conv_bn_relu-415             [-1, 64, 7, 7]               0\n",
      "          Conv2d-416             [-1, 16, 7, 7]           9,216\n",
      "     BatchNorm2d-417             [-1, 16, 7, 7]              32\n",
      "            ReLU-418             [-1, 16, 7, 7]               0\n",
      "    conv_bn_relu-419             [-1, 16, 7, 7]               0\n",
      "          Conv2d-420             [-1, 64, 7, 7]          38,912\n",
      "     BatchNorm2d-421             [-1, 64, 7, 7]             128\n",
      "            ReLU-422             [-1, 64, 7, 7]               0\n",
      "    conv_bn_relu-423             [-1, 64, 7, 7]               0\n",
      "          Conv2d-424             [-1, 16, 7, 7]           9,216\n",
      "     BatchNorm2d-425             [-1, 16, 7, 7]              32\n",
      "            ReLU-426             [-1, 16, 7, 7]               0\n",
      "    conv_bn_relu-427             [-1, 16, 7, 7]               0\n",
      "          Conv2d-428             [-1, 16, 7, 7]           2,304\n",
      "     BatchNorm2d-429             [-1, 16, 7, 7]              32\n",
      "            ReLU-430             [-1, 16, 7, 7]               0\n",
      "    conv_bn_relu-431             [-1, 16, 7, 7]               0\n",
      "     dense_layer-432            [-1, 640, 7, 7]               0\n",
      "          Conv2d-433             [-1, 64, 7, 7]          40,960\n",
      "     BatchNorm2d-434             [-1, 64, 7, 7]             128\n",
      "            ReLU-435             [-1, 64, 7, 7]               0\n",
      "    conv_bn_relu-436             [-1, 64, 7, 7]               0\n",
      "          Conv2d-437             [-1, 16, 7, 7]           9,216\n",
      "     BatchNorm2d-438             [-1, 16, 7, 7]              32\n",
      "            ReLU-439             [-1, 16, 7, 7]               0\n",
      "    conv_bn_relu-440             [-1, 16, 7, 7]               0\n",
      "          Conv2d-441             [-1, 64, 7, 7]          40,960\n",
      "     BatchNorm2d-442             [-1, 64, 7, 7]             128\n",
      "            ReLU-443             [-1, 64, 7, 7]               0\n",
      "    conv_bn_relu-444             [-1, 64, 7, 7]               0\n",
      "          Conv2d-445             [-1, 16, 7, 7]           9,216\n",
      "     BatchNorm2d-446             [-1, 16, 7, 7]              32\n",
      "            ReLU-447             [-1, 16, 7, 7]               0\n",
      "    conv_bn_relu-448             [-1, 16, 7, 7]               0\n",
      "          Conv2d-449             [-1, 16, 7, 7]           2,304\n",
      "     BatchNorm2d-450             [-1, 16, 7, 7]              32\n",
      "            ReLU-451             [-1, 16, 7, 7]               0\n",
      "    conv_bn_relu-452             [-1, 16, 7, 7]               0\n",
      "     dense_layer-453            [-1, 672, 7, 7]               0\n",
      "          Conv2d-454             [-1, 64, 7, 7]          43,008\n",
      "     BatchNorm2d-455             [-1, 64, 7, 7]             128\n",
      "            ReLU-456             [-1, 64, 7, 7]               0\n",
      "    conv_bn_relu-457             [-1, 64, 7, 7]               0\n",
      "          Conv2d-458             [-1, 16, 7, 7]           9,216\n",
      "     BatchNorm2d-459             [-1, 16, 7, 7]              32\n",
      "            ReLU-460             [-1, 16, 7, 7]               0\n",
      "    conv_bn_relu-461             [-1, 16, 7, 7]               0\n",
      "          Conv2d-462             [-1, 64, 7, 7]          43,008\n",
      "     BatchNorm2d-463             [-1, 64, 7, 7]             128\n",
      "            ReLU-464             [-1, 64, 7, 7]               0\n",
      "    conv_bn_relu-465             [-1, 64, 7, 7]               0\n",
      "          Conv2d-466             [-1, 16, 7, 7]           9,216\n",
      "     BatchNorm2d-467             [-1, 16, 7, 7]              32\n",
      "            ReLU-468             [-1, 16, 7, 7]               0\n",
      "    conv_bn_relu-469             [-1, 16, 7, 7]               0\n",
      "          Conv2d-470             [-1, 16, 7, 7]           2,304\n",
      "     BatchNorm2d-471             [-1, 16, 7, 7]              32\n",
      "            ReLU-472             [-1, 16, 7, 7]               0\n",
      "    conv_bn_relu-473             [-1, 16, 7, 7]               0\n",
      "     dense_layer-474            [-1, 704, 7, 7]               0\n",
      "          Conv2d-475            [-1, 704, 7, 7]         495,616\n",
      "     BatchNorm2d-476            [-1, 704, 7, 7]           1,408\n",
      "            ReLU-477            [-1, 704, 7, 7]               0\n",
      "    conv_bn_relu-478            [-1, 704, 7, 7]               0\n",
      "          Linear-479                   [-1, 10]           7,050\n",
      "================================================================\n",
      "Total params: 2,247,210\n",
      "Trainable params: 2,247,210\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.57\n",
      "Forward/backward pass size (MB): 145.29\n",
      "Params size (MB): 8.57\n",
      "Estimated Total Size (MB): 154.44\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "torchsummary.summary(net, (3, 224, 224))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1955
    },
    "colab_type": "code",
    "id": "NbQebd0Ixmaf",
    "outputId": "f272377a-c530-4021-c683-3da80f3b0a77"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1,  6400/50000] loss: 2.0496658, lr: 0.1000000\n",
      "[1, 12800/50000] loss: 1.8068226, lr: 0.1000000\n",
      "[1, 19200/50000] loss: 1.6786690, lr: 0.1000000\n",
      "[1, 25600/50000] loss: 1.5848522, lr: 0.1000000\n",
      "[1, 32000/50000] loss: 1.5190959, lr: 0.1000000\n",
      "[1, 38400/50000] loss: 1.4203354, lr: 0.1000000\n",
      "[1, 44800/50000] loss: 1.3563626, lr: 0.1000000\n",
      "[1 epoch] Accuracy of the network on the validation images: 51 %\n",
      "[2,  6400/50000] loss: 1.2689225, lr: 0.0999753\n",
      "[2, 12800/50000] loss: 1.1836578, lr: 0.0999753\n",
      "[2, 19200/50000] loss: 1.1128522, lr: 0.0999753\n",
      "[2, 25600/50000] loss: 1.0888133, lr: 0.0999753\n",
      "[2, 32000/50000] loss: 1.0325753, lr: 0.0999753\n",
      "[2, 38400/50000] loss: 0.9965671, lr: 0.0999753\n",
      "[2, 44800/50000] loss: 0.9671038, lr: 0.0999753\n",
      "[2 epoch] Accuracy of the network on the validation images: 67 %\n",
      "[3,  6400/50000] loss: 0.9065573, lr: 0.0999013\n",
      "[3, 12800/50000] loss: 0.8651605, lr: 0.0999013\n",
      "[3, 19200/50000] loss: 0.8328243, lr: 0.0999013\n",
      "[3, 25600/50000] loss: 0.8386372, lr: 0.0999013\n",
      "[3, 32000/50000] loss: 0.8222058, lr: 0.0999013\n",
      "[3, 38400/50000] loss: 0.8085056, lr: 0.0999013\n",
      "[3, 44800/50000] loss: 0.7698177, lr: 0.0999013\n",
      "[3 epoch] Accuracy of the network on the validation images: 72 %\n",
      "[4,  6400/50000] loss: 0.7291356, lr: 0.0997781\n",
      "[4, 12800/50000] loss: 0.7372426, lr: 0.0997781\n",
      "[4, 19200/50000] loss: 0.7058966, lr: 0.0997781\n",
      "[4, 25600/50000] loss: 0.7049237, lr: 0.0997781\n",
      "[4, 32000/50000] loss: 0.6676211, lr: 0.0997781\n",
      "[4, 38400/50000] loss: 0.6593976, lr: 0.0997781\n",
      "[4, 44800/50000] loss: 0.6505711, lr: 0.0997781\n",
      "[4 epoch] Accuracy of the network on the validation images: 77 %\n",
      "[5,  6400/50000] loss: 0.5931259, lr: 0.0996057\n",
      "[5, 12800/50000] loss: 0.5752838, lr: 0.0996057\n",
      "[5, 19200/50000] loss: 0.6119232, lr: 0.0996057\n",
      "[5, 25600/50000] loss: 0.6128264, lr: 0.0996057\n",
      "[5, 32000/50000] loss: 0.5792050, lr: 0.0996057\n",
      "[5, 38400/50000] loss: 0.5565957, lr: 0.0996057\n",
      "[5, 44800/50000] loss: 0.5623935, lr: 0.0996057\n",
      "[5 epoch] Accuracy of the network on the validation images: 81 %\n",
      "[6,  6400/50000] loss: 0.5166937, lr: 0.0993844\n",
      "[6, 12800/50000] loss: 0.5271167, lr: 0.0993844\n",
      "[6, 19200/50000] loss: 0.4984480, lr: 0.0993844\n",
      "[6, 25600/50000] loss: 0.5035617, lr: 0.0993844\n",
      "[6, 32000/50000] loss: 0.5007112, lr: 0.0993844\n",
      "[6, 38400/50000] loss: 0.4927954, lr: 0.0993844\n",
      "[6, 44800/50000] loss: 0.5092412, lr: 0.0993844\n",
      "[6 epoch] Accuracy of the network on the validation images: 81 %\n",
      "[7,  6400/50000] loss: 0.4527915, lr: 0.0991144\n",
      "[7, 12800/50000] loss: 0.4400892, lr: 0.0991144\n",
      "[7, 19200/50000] loss: 0.4735666, lr: 0.0991144\n",
      "[7, 25600/50000] loss: 0.4420315, lr: 0.0991144\n",
      "[7, 32000/50000] loss: 0.4523846, lr: 0.0991144\n",
      "[7, 38400/50000] loss: 0.4447765, lr: 0.0991144\n",
      "[7, 44800/50000] loss: 0.4326478, lr: 0.0991144\n",
      "[7 epoch] Accuracy of the network on the validation images: 82 %\n",
      "[8,  6400/50000] loss: 0.4169219, lr: 0.0987958\n",
      "[8, 12800/50000] loss: 0.4240849, lr: 0.0987958\n",
      "[8, 19200/50000] loss: 0.3974233, lr: 0.0987958\n",
      "[8, 25600/50000] loss: 0.4046972, lr: 0.0987958\n",
      "[8, 32000/50000] loss: 0.4117470, lr: 0.0987958\n",
      "[8, 38400/50000] loss: 0.4092274, lr: 0.0987958\n",
      "[8, 44800/50000] loss: 0.4071747, lr: 0.0987958\n",
      "[8 epoch] Accuracy of the network on the validation images: 84 %\n",
      "[9,  6400/50000] loss: 0.3494892, lr: 0.0984292\n",
      "[9, 12800/50000] loss: 0.3753338, lr: 0.0984292\n",
      "[9, 19200/50000] loss: 0.3671228, lr: 0.0984292\n",
      "[9, 25600/50000] loss: 0.3732030, lr: 0.0984292\n",
      "[9, 32000/50000] loss: 0.3796470, lr: 0.0984292\n",
      "[9, 38400/50000] loss: 0.3640567, lr: 0.0984292\n",
      "[9, 44800/50000] loss: 0.3783031, lr: 0.0984292\n",
      "[9 epoch] Accuracy of the network on the validation images: 83 %\n",
      "[10,  6400/50000] loss: 0.3481686, lr: 0.0980147\n",
      "[10, 12800/50000] loss: 0.3285683, lr: 0.0980147\n",
      "[10, 19200/50000] loss: 0.3366911, lr: 0.0980147\n",
      "[10, 25600/50000] loss: 0.3431504, lr: 0.0980147\n",
      "[10, 32000/50000] loss: 0.3387421, lr: 0.0980147\n",
      "[10, 38400/50000] loss: 0.3404589, lr: 0.0980147\n",
      "[10, 44800/50000] loss: 0.3318516, lr: 0.0980147\n",
      "[10 epoch] Accuracy of the network on the validation images: 86 %\n",
      "[11,  6400/50000] loss: 0.3036378, lr: 0.0975528\n",
      "[11, 12800/50000] loss: 0.3192177, lr: 0.0975528\n",
      "[11, 19200/50000] loss: 0.3102059, lr: 0.0975528\n",
      "[11, 25600/50000] loss: 0.3051294, lr: 0.0975528\n",
      "[11, 32000/50000] loss: 0.3249552, lr: 0.0975528\n",
      "[11, 38400/50000] loss: 0.3055301, lr: 0.0975528\n",
      "[11, 44800/50000] loss: 0.3216736, lr: 0.0975528\n",
      "[11 epoch] Accuracy of the network on the validation images: 85 %\n",
      "[12,  6400/50000] loss: 0.2800993, lr: 0.0970440\n",
      "[12, 12800/50000] loss: 0.3031481, lr: 0.0970440\n",
      "[12, 19200/50000] loss: 0.2757101, lr: 0.0970440\n",
      "[12, 25600/50000] loss: 0.2799333, lr: 0.0970440\n",
      "[12, 32000/50000] loss: 0.2832145, lr: 0.0970440\n",
      "[12, 38400/50000] loss: 0.2822161, lr: 0.0970440\n",
      "[12, 44800/50000] loss: 0.2935896, lr: 0.0970440\n",
      "[12 epoch] Accuracy of the network on the validation images: 87 %\n",
      "[13,  6400/50000] loss: 0.2450171, lr: 0.0964888\n",
      "[13, 12800/50000] loss: 0.2632875, lr: 0.0964888\n",
      "[13, 19200/50000] loss: 0.2493772, lr: 0.0964888\n",
      "[13, 25600/50000] loss: 0.2557572, lr: 0.0964888\n",
      "[13, 32000/50000] loss: 0.2729698, lr: 0.0964888\n",
      "[13, 38400/50000] loss: 0.2837593, lr: 0.0964888\n",
      "[13, 44800/50000] loss: 0.2682040, lr: 0.0964888\n",
      "[13 epoch] Accuracy of the network on the validation images: 86 %\n",
      "[14,  6400/50000] loss: 0.2438732, lr: 0.0958877\n",
      "[14, 12800/50000] loss: 0.2614708, lr: 0.0958877\n",
      "[14, 19200/50000] loss: 0.2471270, lr: 0.0958877\n",
      "[14, 25600/50000] loss: 0.2443577, lr: 0.0958877\n",
      "[14, 32000/50000] loss: 0.2499487, lr: 0.0958877\n",
      "[14, 38400/50000] loss: 0.2418676, lr: 0.0958877\n",
      "[14, 44800/50000] loss: 0.2565013, lr: 0.0958877\n",
      "[14 epoch] Accuracy of the network on the validation images: 87 %\n",
      "[15,  6400/50000] loss: 0.2276111, lr: 0.0952414\n",
      "[15, 12800/50000] loss: 0.2034573, lr: 0.0952414\n",
      "[15, 19200/50000] loss: 0.2190720, lr: 0.0952414\n",
      "[15, 25600/50000] loss: 0.2430209, lr: 0.0952414\n",
      "[15, 32000/50000] loss: 0.2316632, lr: 0.0952414\n",
      "[15, 38400/50000] loss: 0.2396084, lr: 0.0952414\n",
      "[15, 44800/50000] loss: 0.2213777, lr: 0.0952414\n",
      "[15 epoch] Accuracy of the network on the validation images: 87 %\n",
      "[16,  6400/50000] loss: 0.2084038, lr: 0.0945503\n",
      "[16, 12800/50000] loss: 0.2066515, lr: 0.0945503\n",
      "[16, 19200/50000] loss: 0.2236241, lr: 0.0945503\n",
      "[16, 25600/50000] loss: 0.2136224, lr: 0.0945503\n",
      "[16, 32000/50000] loss: 0.2151919, lr: 0.0945503\n",
      "[16, 38400/50000] loss: 0.2315261, lr: 0.0945503\n",
      "[16, 44800/50000] loss: 0.2167219, lr: 0.0945503\n",
      "[16 epoch] Accuracy of the network on the validation images: 89 %\n",
      "[17,  6400/50000] loss: 0.1760298, lr: 0.0938153\n",
      "[17, 12800/50000] loss: 0.1873789, lr: 0.0938153\n",
      "[17, 19200/50000] loss: 0.1845537, lr: 0.0938153\n",
      "[17, 25600/50000] loss: 0.1873605, lr: 0.0938153\n",
      "[17, 32000/50000] loss: 0.1953586, lr: 0.0938153\n",
      "[17, 38400/50000] loss: 0.2148755, lr: 0.0938153\n",
      "[17, 44800/50000] loss: 0.1916543, lr: 0.0938153\n",
      "[17 epoch] Accuracy of the network on the validation images: 88 %\n",
      "[18,  6400/50000] loss: 0.1868909, lr: 0.0930371\n",
      "[18, 12800/50000] loss: 0.1834016, lr: 0.0930371\n",
      "[18, 19200/50000] loss: 0.1932407, lr: 0.0930371\n",
      "[18, 25600/50000] loss: 0.1796101, lr: 0.0930371\n",
      "[18, 32000/50000] loss: 0.1737827, lr: 0.0930371\n",
      "[18, 38400/50000] loss: 0.1854231, lr: 0.0930371\n",
      "[18, 44800/50000] loss: 0.1991799, lr: 0.0930371\n",
      "[18 epoch] Accuracy of the network on the validation images: 87 %\n",
      "[19,  6400/50000] loss: 0.1600758, lr: 0.0922164\n",
      "[19, 12800/50000] loss: 0.1518982, lr: 0.0922164\n",
      "[19, 19200/50000] loss: 0.1555410, lr: 0.0922164\n",
      "[19, 25600/50000] loss: 0.1627056, lr: 0.0922164\n",
      "[19, 32000/50000] loss: 0.1731801, lr: 0.0922164\n",
      "[19, 38400/50000] loss: 0.1915393, lr: 0.0922164\n",
      "[19, 44800/50000] loss: 0.1852191, lr: 0.0922164\n",
      "[19 epoch] Accuracy of the network on the validation images: 87 %\n",
      "[20,  6400/50000] loss: 0.1426546, lr: 0.0913540\n",
      "[20, 12800/50000] loss: 0.1331994, lr: 0.0913540\n",
      "[20, 19200/50000] loss: 0.1521909, lr: 0.0913540\n",
      "[20, 25600/50000] loss: 0.1667132, lr: 0.0913540\n",
      "[20, 32000/50000] loss: 0.1567183, lr: 0.0913540\n",
      "[20, 38400/50000] loss: 0.1622897, lr: 0.0913540\n",
      "[20, 44800/50000] loss: 0.1761142, lr: 0.0913540\n",
      "[20 epoch] Accuracy of the network on the validation images: 88 %\n",
      "[21,  6400/50000] loss: 0.1903395, lr: 0.0904508\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[21, 12800/50000] loss: 0.1413518, lr: 0.0904508\n",
      "[21, 19200/50000] loss: 0.1452274, lr: 0.0904508\n",
      "[21, 25600/50000] loss: 0.1475782, lr: 0.0904508\n",
      "[21, 32000/50000] loss: 0.1456229, lr: 0.0904508\n",
      "[21, 38400/50000] loss: 0.1574638, lr: 0.0904508\n",
      "[21, 44800/50000] loss: 0.1524778, lr: 0.0904508\n",
      "[21 epoch] Accuracy of the network on the validation images: 88 %\n",
      "[22,  6400/50000] loss: 0.1222139, lr: 0.0895078\n",
      "[22, 12800/50000] loss: 0.1358480, lr: 0.0895078\n",
      "[22, 19200/50000] loss: 0.1337867, lr: 0.0895078\n",
      "[22, 25600/50000] loss: 0.1315738, lr: 0.0895078\n",
      "[22, 32000/50000] loss: 0.1434432, lr: 0.0895078\n",
      "[22, 38400/50000] loss: 0.1534713, lr: 0.0895078\n",
      "[22, 44800/50000] loss: 0.1387539, lr: 0.0895078\n",
      "[22 epoch] Accuracy of the network on the validation images: 89 %\n",
      "[23,  6400/50000] loss: 0.1525227, lr: 0.0885257\n",
      "[23, 12800/50000] loss: 0.1344277, lr: 0.0885257\n",
      "[23, 19200/50000] loss: 0.1197590, lr: 0.0885257\n",
      "[23, 25600/50000] loss: 0.1392264, lr: 0.0885257\n",
      "[23, 32000/50000] loss: 0.1197026, lr: 0.0885257\n",
      "[23, 38400/50000] loss: 0.1331452, lr: 0.0885257\n",
      "[23, 44800/50000] loss: 0.1300447, lr: 0.0885257\n",
      "[23 epoch] Accuracy of the network on the validation images: 88 %\n",
      "[24,  6400/50000] loss: 0.1172571, lr: 0.0875056\n",
      "[24, 12800/50000] loss: 0.1100674, lr: 0.0875056\n",
      "[24, 19200/50000] loss: 0.1097740, lr: 0.0875056\n",
      "[24, 25600/50000] loss: 0.1125136, lr: 0.0875056\n",
      "[24, 32000/50000] loss: 0.1127336, lr: 0.0875056\n",
      "[24, 38400/50000] loss: 0.1226763, lr: 0.0875056\n",
      "[24, 44800/50000] loss: 0.1201403, lr: 0.0875056\n",
      "[24 epoch] Accuracy of the network on the validation images: 88 %\n",
      "[25,  6400/50000] loss: 0.1189362, lr: 0.0864484\n",
      "[25, 12800/50000] loss: 0.1102018, lr: 0.0864484\n",
      "[25, 19200/50000] loss: 0.1166698, lr: 0.0864484\n",
      "[25, 25600/50000] loss: 0.1116896, lr: 0.0864484\n",
      "[25, 32000/50000] loss: 0.1170792, lr: 0.0864484\n",
      "[25, 38400/50000] loss: 0.1206930, lr: 0.0864484\n",
      "[25, 44800/50000] loss: 0.1189561, lr: 0.0864484\n",
      "[25 epoch] Accuracy of the network on the validation images: 89 %\n",
      "[26,  6400/50000] loss: 0.1151441, lr: 0.0853553\n",
      "[26, 12800/50000] loss: 0.1064161, lr: 0.0853553\n",
      "[26, 19200/50000] loss: 0.1039228, lr: 0.0853553\n",
      "[26, 25600/50000] loss: 0.0959971, lr: 0.0853553\n",
      "[26, 32000/50000] loss: 0.0942086, lr: 0.0853553\n",
      "[26, 38400/50000] loss: 0.0999814, lr: 0.0853553\n",
      "[26, 44800/50000] loss: 0.1071462, lr: 0.0853553\n",
      "[26 epoch] Accuracy of the network on the validation images: 89 %\n",
      "[27,  6400/50000] loss: 0.0863350, lr: 0.0842274\n",
      "[27, 12800/50000] loss: 0.0744723, lr: 0.0842274\n",
      "[27, 19200/50000] loss: 0.0989238, lr: 0.0842274\n",
      "[27, 25600/50000] loss: 0.0898941, lr: 0.0842274\n",
      "[27, 32000/50000] loss: 0.0908760, lr: 0.0842274\n",
      "[27, 38400/50000] loss: 0.0926984, lr: 0.0842274\n",
      "[27, 44800/50000] loss: 0.0911031, lr: 0.0842274\n",
      "[27 epoch] Accuracy of the network on the validation images: 89 %\n",
      "[28,  6400/50000] loss: 0.0813437, lr: 0.0830656\n",
      "[28, 12800/50000] loss: 0.0785445, lr: 0.0830656\n",
      "[28, 19200/50000] loss: 0.0825007, lr: 0.0830656\n",
      "[28, 25600/50000] loss: 0.0943294, lr: 0.0830656\n",
      "[28, 32000/50000] loss: 0.0822389, lr: 0.0830656\n",
      "[28, 38400/50000] loss: 0.0996517, lr: 0.0830656\n",
      "[28, 44800/50000] loss: 0.0991702, lr: 0.0830656\n",
      "[28 epoch] Accuracy of the network on the validation images: 89 %\n",
      "[29,  6400/50000] loss: 0.0721990, lr: 0.0818712\n",
      "[29, 12800/50000] loss: 0.0747066, lr: 0.0818712\n",
      "[29, 19200/50000] loss: 0.0681735, lr: 0.0818712\n",
      "[29, 25600/50000] loss: 0.0800962, lr: 0.0818712\n",
      "[29, 32000/50000] loss: 0.0740597, lr: 0.0818712\n",
      "[29, 38400/50000] loss: 0.0828522, lr: 0.0818712\n",
      "[29, 44800/50000] loss: 0.0872220, lr: 0.0818712\n",
      "[29 epoch] Accuracy of the network on the validation images: 89 %\n",
      "[30,  6400/50000] loss: 0.0928636, lr: 0.0806454\n",
      "[30, 12800/50000] loss: 0.0793017, lr: 0.0806454\n",
      "[30, 19200/50000] loss: 0.0786104, lr: 0.0806454\n",
      "[30, 25600/50000] loss: 0.0650846, lr: 0.0806454\n",
      "[30, 32000/50000] loss: 0.0764670, lr: 0.0806454\n",
      "[30, 38400/50000] loss: 0.0829891, lr: 0.0806454\n",
      "[30, 44800/50000] loss: 0.0804977, lr: 0.0806454\n",
      "[30 epoch] Accuracy of the network on the validation images: 89 %\n",
      "[31,  6400/50000] loss: 0.0617269, lr: 0.0793893\n",
      "[31, 12800/50000] loss: 0.0635441, lr: 0.0793893\n",
      "[31, 19200/50000] loss: 0.0692841, lr: 0.0793893\n",
      "[31, 25600/50000] loss: 0.0570056, lr: 0.0793893\n",
      "[31, 32000/50000] loss: 0.0729992, lr: 0.0793893\n",
      "[31, 38400/50000] loss: 0.0776230, lr: 0.0793893\n",
      "[31, 44800/50000] loss: 0.0709974, lr: 0.0793893\n",
      "[31 epoch] Accuracy of the network on the validation images: 89 %\n",
      "[32,  6400/50000] loss: 0.0920748, lr: 0.0781042\n",
      "[32, 12800/50000] loss: 0.0646205, lr: 0.0781042\n",
      "[32, 19200/50000] loss: 0.0584229, lr: 0.0781042\n",
      "[32, 25600/50000] loss: 0.0692199, lr: 0.0781042\n",
      "[32, 32000/50000] loss: 0.0690325, lr: 0.0781042\n",
      "[32, 38400/50000] loss: 0.0734952, lr: 0.0781042\n",
      "[32, 44800/50000] loss: 0.0653615, lr: 0.0781042\n",
      "[32 epoch] Accuracy of the network on the validation images: 89 %\n",
      "[33,  6400/50000] loss: 0.0639692, lr: 0.0767913\n",
      "[33, 12800/50000] loss: 0.0527627, lr: 0.0767913\n",
      "[33, 19200/50000] loss: 0.0505792, lr: 0.0767913\n",
      "[33, 25600/50000] loss: 0.0627917, lr: 0.0767913\n",
      "[33, 32000/50000] loss: 0.0606606, lr: 0.0767913\n",
      "[33, 38400/50000] loss: 0.0561571, lr: 0.0767913\n",
      "[33, 44800/50000] loss: 0.0714912, lr: 0.0767913\n",
      "[33 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[34,  6400/50000] loss: 0.0897149, lr: 0.0754521\n",
      "[34, 12800/50000] loss: 0.0662515, lr: 0.0754521\n",
      "[34, 19200/50000] loss: 0.0582368, lr: 0.0754521\n",
      "[34, 25600/50000] loss: 0.0570027, lr: 0.0754521\n",
      "[34, 32000/50000] loss: 0.0554830, lr: 0.0754521\n",
      "[34, 38400/50000] loss: 0.0587204, lr: 0.0754521\n",
      "[34, 44800/50000] loss: 0.0681816, lr: 0.0754521\n",
      "[34 epoch] Accuracy of the network on the validation images: 89 %\n",
      "[35,  6400/50000] loss: 0.0618721, lr: 0.0740877\n",
      "[35, 12800/50000] loss: 0.0560294, lr: 0.0740877\n",
      "[35, 19200/50000] loss: 0.0557079, lr: 0.0740877\n",
      "[35, 25600/50000] loss: 0.0571260, lr: 0.0740877\n",
      "[35, 32000/50000] loss: 0.0469641, lr: 0.0740877\n",
      "[35, 38400/50000] loss: 0.0676147, lr: 0.0740877\n",
      "[35, 44800/50000] loss: 0.0499982, lr: 0.0740877\n",
      "[35 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[36,  6400/50000] loss: 0.0686994, lr: 0.0726995\n",
      "[36, 12800/50000] loss: 0.0508266, lr: 0.0726995\n",
      "[36, 19200/50000] loss: 0.0488176, lr: 0.0726995\n",
      "[36, 25600/50000] loss: 0.0487924, lr: 0.0726995\n",
      "[36, 32000/50000] loss: 0.0495834, lr: 0.0726995\n",
      "[36, 38400/50000] loss: 0.0485713, lr: 0.0726995\n",
      "[36, 44800/50000] loss: 0.0535082, lr: 0.0726995\n",
      "[36 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[37,  6400/50000] loss: 0.0413086, lr: 0.0712890\n",
      "[37, 12800/50000] loss: 0.0380331, lr: 0.0712890\n",
      "[37, 19200/50000] loss: 0.0464289, lr: 0.0712890\n",
      "[37, 25600/50000] loss: 0.0432421, lr: 0.0712890\n",
      "[37, 32000/50000] loss: 0.0445515, lr: 0.0712890\n",
      "[37, 38400/50000] loss: 0.0442440, lr: 0.0712890\n",
      "[37, 44800/50000] loss: 0.0416149, lr: 0.0712890\n",
      "[37 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[38,  6400/50000] loss: 0.0620349, lr: 0.0698574\n",
      "[38, 12800/50000] loss: 0.0448007, lr: 0.0698574\n",
      "[38, 19200/50000] loss: 0.0392395, lr: 0.0698574\n",
      "[38, 25600/50000] loss: 0.0428438, lr: 0.0698574\n",
      "[38, 32000/50000] loss: 0.0323482, lr: 0.0698574\n",
      "[38, 38400/50000] loss: 0.0450538, lr: 0.0698574\n",
      "[38, 44800/50000] loss: 0.0366678, lr: 0.0698574\n",
      "[38 epoch] Accuracy of the network on the validation images: 89 %\n",
      "[39,  6400/50000] loss: 0.0575658, lr: 0.0684062\n",
      "[39, 12800/50000] loss: 0.0357294, lr: 0.0684062\n",
      "[39, 19200/50000] loss: 0.0363159, lr: 0.0684062\n",
      "[39, 25600/50000] loss: 0.0356628, lr: 0.0684062\n",
      "[39, 32000/50000] loss: 0.0335677, lr: 0.0684062\n",
      "[39, 38400/50000] loss: 0.0341221, lr: 0.0684062\n",
      "[39, 44800/50000] loss: 0.0422140, lr: 0.0684062\n",
      "[39 epoch] Accuracy of the network on the validation images: 89 %\n",
      "[40,  6400/50000] loss: 0.0344810, lr: 0.0669369\n",
      "[40, 12800/50000] loss: 0.0310291, lr: 0.0669369\n",
      "[40, 19200/50000] loss: 0.0317852, lr: 0.0669369\n",
      "[40, 25600/50000] loss: 0.0351711, lr: 0.0669369\n",
      "[40, 32000/50000] loss: 0.0335717, lr: 0.0669369\n",
      "[40, 38400/50000] loss: 0.0351966, lr: 0.0669369\n",
      "[40, 44800/50000] loss: 0.0332071, lr: 0.0669369\n",
      "[40 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[41,  6400/50000] loss: 0.0279484, lr: 0.0654508\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[41, 12800/50000] loss: 0.0315345, lr: 0.0654508\n",
      "[41, 19200/50000] loss: 0.0282899, lr: 0.0654508\n",
      "[41, 25600/50000] loss: 0.0343004, lr: 0.0654508\n",
      "[41, 32000/50000] loss: 0.0250815, lr: 0.0654508\n",
      "[41, 38400/50000] loss: 0.0263588, lr: 0.0654508\n",
      "[41, 44800/50000] loss: 0.0288901, lr: 0.0654508\n",
      "[41 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[42,  6400/50000] loss: 0.0240217, lr: 0.0639496\n",
      "[42, 12800/50000] loss: 0.0222370, lr: 0.0639496\n",
      "[42, 19200/50000] loss: 0.0224188, lr: 0.0639496\n",
      "[42, 25600/50000] loss: 0.0290490, lr: 0.0639496\n",
      "[42, 32000/50000] loss: 0.0304564, lr: 0.0639496\n",
      "[42, 38400/50000] loss: 0.0314656, lr: 0.0639496\n",
      "[42, 44800/50000] loss: 0.0291475, lr: 0.0639496\n",
      "[42 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[43,  6400/50000] loss: 0.0342173, lr: 0.0624345\n",
      "[43, 12800/50000] loss: 0.0291759, lr: 0.0624345\n",
      "[43, 19200/50000] loss: 0.0262700, lr: 0.0624345\n",
      "[43, 25600/50000] loss: 0.0262725, lr: 0.0624345\n",
      "[43, 32000/50000] loss: 0.0269235, lr: 0.0624345\n",
      "[43, 38400/50000] loss: 0.0257980, lr: 0.0624345\n",
      "[43, 44800/50000] loss: 0.0267782, lr: 0.0624345\n",
      "[43 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[44,  6400/50000] loss: 0.0224463, lr: 0.0609072\n",
      "[44, 12800/50000] loss: 0.0200076, lr: 0.0609072\n",
      "[44, 19200/50000] loss: 0.0256564, lr: 0.0609072\n",
      "[44, 25600/50000] loss: 0.0218179, lr: 0.0609072\n",
      "[44, 32000/50000] loss: 0.0205171, lr: 0.0609072\n",
      "[44, 38400/50000] loss: 0.0239712, lr: 0.0609072\n",
      "[44, 44800/50000] loss: 0.0239041, lr: 0.0609072\n",
      "[44 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[45,  6400/50000] loss: 0.0392942, lr: 0.0593691\n",
      "[45, 12800/50000] loss: 0.0295278, lr: 0.0593691\n",
      "[45, 19200/50000] loss: 0.0235677, lr: 0.0593691\n",
      "[45, 25600/50000] loss: 0.0295603, lr: 0.0593691\n",
      "[45, 32000/50000] loss: 0.0273553, lr: 0.0593691\n",
      "[45, 38400/50000] loss: 0.0285113, lr: 0.0593691\n",
      "[45, 44800/50000] loss: 0.0305550, lr: 0.0593691\n",
      "[45 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[46,  6400/50000] loss: 0.0274109, lr: 0.0578217\n",
      "[46, 12800/50000] loss: 0.0189200, lr: 0.0578217\n",
      "[46, 19200/50000] loss: 0.0229468, lr: 0.0578217\n",
      "[46, 25600/50000] loss: 0.0219984, lr: 0.0578217\n",
      "[46, 32000/50000] loss: 0.0233198, lr: 0.0578217\n",
      "[46, 38400/50000] loss: 0.0279039, lr: 0.0578217\n",
      "[46, 44800/50000] loss: 0.0194732, lr: 0.0578217\n",
      "[46 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[47,  6400/50000] loss: 0.0202283, lr: 0.0562667\n",
      "[47, 12800/50000] loss: 0.0167560, lr: 0.0562667\n",
      "[47, 19200/50000] loss: 0.0198608, lr: 0.0562667\n",
      "[47, 25600/50000] loss: 0.0198559, lr: 0.0562667\n",
      "[47, 32000/50000] loss: 0.0234401, lr: 0.0562667\n",
      "[47, 38400/50000] loss: 0.0211893, lr: 0.0562667\n",
      "[47, 44800/50000] loss: 0.0178531, lr: 0.0562667\n",
      "[47 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[48,  6400/50000] loss: 0.0402080, lr: 0.0547054\n",
      "[48, 12800/50000] loss: 0.0267559, lr: 0.0547054\n",
      "[48, 19200/50000] loss: 0.0220693, lr: 0.0547054\n",
      "[48, 25600/50000] loss: 0.0205541, lr: 0.0547054\n",
      "[48, 32000/50000] loss: 0.0176802, lr: 0.0547054\n",
      "[48, 38400/50000] loss: 0.0235914, lr: 0.0547054\n",
      "[48, 44800/50000] loss: 0.0197932, lr: 0.0547054\n",
      "[48 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[49,  6400/50000] loss: 0.0195776, lr: 0.0531395\n",
      "[49, 12800/50000] loss: 0.0159212, lr: 0.0531395\n",
      "[49, 19200/50000] loss: 0.0161382, lr: 0.0531395\n",
      "[49, 25600/50000] loss: 0.0145114, lr: 0.0531395\n",
      "[49, 32000/50000] loss: 0.0168772, lr: 0.0531395\n",
      "[49, 38400/50000] loss: 0.0154078, lr: 0.0531395\n",
      "[49, 44800/50000] loss: 0.0174041, lr: 0.0531395\n",
      "[49 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[50,  6400/50000] loss: 0.0125563, lr: 0.0515705\n",
      "[50, 12800/50000] loss: 0.0122453, lr: 0.0515705\n",
      "[50, 19200/50000] loss: 0.0136951, lr: 0.0515705\n",
      "[50, 25600/50000] loss: 0.0148767, lr: 0.0515705\n",
      "[50, 32000/50000] loss: 0.0118229, lr: 0.0515705\n",
      "[50, 38400/50000] loss: 0.0187339, lr: 0.0515705\n",
      "[50, 44800/50000] loss: 0.0146875, lr: 0.0515705\n",
      "[50 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[51,  6400/50000] loss: 0.0099551, lr: 0.0500000\n",
      "[51, 12800/50000] loss: 0.0111863, lr: 0.0500000\n",
      "[51, 19200/50000] loss: 0.0125809, lr: 0.0500000\n",
      "[51, 25600/50000] loss: 0.0115431, lr: 0.0500000\n",
      "[51, 32000/50000] loss: 0.0105551, lr: 0.0500000\n",
      "[51, 38400/50000] loss: 0.0156531, lr: 0.0500000\n",
      "[51, 44800/50000] loss: 0.0153766, lr: 0.0500000\n",
      "[51 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[52,  6400/50000] loss: 0.0100791, lr: 0.0484295\n",
      "[52, 12800/50000] loss: 0.0100674, lr: 0.0484295\n",
      "[52, 19200/50000] loss: 0.0095729, lr: 0.0484295\n",
      "[52, 25600/50000] loss: 0.0096198, lr: 0.0484295\n",
      "[52, 32000/50000] loss: 0.0075502, lr: 0.0484295\n",
      "[52, 38400/50000] loss: 0.0096830, lr: 0.0484295\n",
      "[52, 44800/50000] loss: 0.0101689, lr: 0.0484295\n",
      "[52 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[53,  6400/50000] loss: 0.0111580, lr: 0.0468605\n",
      "[53, 12800/50000] loss: 0.0084673, lr: 0.0468605\n",
      "[53, 19200/50000] loss: 0.0130090, lr: 0.0468605\n",
      "[53, 25600/50000] loss: 0.0102558, lr: 0.0468605\n",
      "[53, 32000/50000] loss: 0.0111842, lr: 0.0468605\n",
      "[53, 38400/50000] loss: 0.0118133, lr: 0.0468605\n",
      "[53, 44800/50000] loss: 0.0110651, lr: 0.0468605\n",
      "[53 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[54,  6400/50000] loss: 0.0083549, lr: 0.0452946\n",
      "[54, 12800/50000] loss: 0.0118107, lr: 0.0452946\n",
      "[54, 19200/50000] loss: 0.0081777, lr: 0.0452946\n",
      "[54, 25600/50000] loss: 0.0080900, lr: 0.0452946\n",
      "[54, 32000/50000] loss: 0.0087536, lr: 0.0452946\n",
      "[54, 38400/50000] loss: 0.0093351, lr: 0.0452946\n",
      "[54, 44800/50000] loss: 0.0083984, lr: 0.0452946\n",
      "[54 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[55,  6400/50000] loss: 0.0088965, lr: 0.0437333\n",
      "[55, 12800/50000] loss: 0.0088393, lr: 0.0437333\n",
      "[55, 19200/50000] loss: 0.0071171, lr: 0.0437333\n",
      "[55, 25600/50000] loss: 0.0097836, lr: 0.0437333\n",
      "[55, 32000/50000] loss: 0.0078453, lr: 0.0437333\n",
      "[55, 38400/50000] loss: 0.0114091, lr: 0.0437333\n",
      "[55, 44800/50000] loss: 0.0094710, lr: 0.0437333\n",
      "[55 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[56,  6400/50000] loss: 0.0072199, lr: 0.0421783\n",
      "[56, 12800/50000] loss: 0.0080935, lr: 0.0421783\n",
      "[56, 19200/50000] loss: 0.0060986, lr: 0.0421783\n",
      "[56, 25600/50000] loss: 0.0075634, lr: 0.0421783\n",
      "[56, 32000/50000] loss: 0.0092670, lr: 0.0421783\n",
      "[56, 38400/50000] loss: 0.0094013, lr: 0.0421783\n",
      "[56, 44800/50000] loss: 0.0081251, lr: 0.0421783\n",
      "[56 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[57,  6400/50000] loss: 0.0138865, lr: 0.0406309\n",
      "[57, 12800/50000] loss: 0.0084548, lr: 0.0406309\n",
      "[57, 19200/50000] loss: 0.0081669, lr: 0.0406309\n",
      "[57, 25600/50000] loss: 0.0094628, lr: 0.0406309\n",
      "[57, 32000/50000] loss: 0.0104640, lr: 0.0406309\n",
      "[57, 38400/50000] loss: 0.0078510, lr: 0.0406309\n",
      "[57, 44800/50000] loss: 0.0066564, lr: 0.0406309\n",
      "[57 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[58,  6400/50000] loss: 0.0205117, lr: 0.0390928\n",
      "[58, 12800/50000] loss: 0.0139543, lr: 0.0390928\n",
      "[58, 19200/50000] loss: 0.0113614, lr: 0.0390928\n",
      "[58, 25600/50000] loss: 0.0106324, lr: 0.0390928\n",
      "[58, 32000/50000] loss: 0.0090071, lr: 0.0390928\n",
      "[58, 38400/50000] loss: 0.0089547, lr: 0.0390928\n",
      "[58, 44800/50000] loss: 0.0102107, lr: 0.0390928\n",
      "[58 epoch] Accuracy of the network on the validation images: 90 %\n",
      "[59,  6400/50000] loss: 0.0081833, lr: 0.0375655\n",
      "[59, 12800/50000] loss: 0.0091810, lr: 0.0375655\n",
      "[59, 19200/50000] loss: 0.0059155, lr: 0.0375655\n",
      "[59, 25600/50000] loss: 0.0068682, lr: 0.0375655\n",
      "[59, 32000/50000] loss: 0.0050115, lr: 0.0375655\n",
      "[59, 38400/50000] loss: 0.0056241, lr: 0.0375655\n",
      "[59, 44800/50000] loss: 0.0050933, lr: 0.0375655\n",
      "[59 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[60,  6400/50000] loss: 0.0058421, lr: 0.0360504\n",
      "[60, 12800/50000] loss: 0.0064903, lr: 0.0360504\n",
      "[60, 19200/50000] loss: 0.0053623, lr: 0.0360504\n",
      "[60, 25600/50000] loss: 0.0060466, lr: 0.0360504\n",
      "[60, 32000/50000] loss: 0.0051920, lr: 0.0360504\n",
      "[60, 38400/50000] loss: 0.0050834, lr: 0.0360504\n",
      "[60, 44800/50000] loss: 0.0053487, lr: 0.0360504\n",
      "[60 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[61,  6400/50000] loss: 0.0044752, lr: 0.0345492\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[61, 12800/50000] loss: 0.0046097, lr: 0.0345492\n",
      "[61, 19200/50000] loss: 0.0058113, lr: 0.0345492\n",
      "[61, 25600/50000] loss: 0.0059015, lr: 0.0345492\n",
      "[61, 32000/50000] loss: 0.0028465, lr: 0.0345492\n",
      "[61, 38400/50000] loss: 0.0056427, lr: 0.0345492\n",
      "[61, 44800/50000] loss: 0.0050590, lr: 0.0345492\n",
      "[61 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[62,  6400/50000] loss: 0.0046700, lr: 0.0330631\n",
      "[62, 12800/50000] loss: 0.0051078, lr: 0.0330631\n",
      "[62, 19200/50000] loss: 0.0036939, lr: 0.0330631\n",
      "[62, 25600/50000] loss: 0.0061979, lr: 0.0330631\n",
      "[62, 32000/50000] loss: 0.0055176, lr: 0.0330631\n",
      "[62, 38400/50000] loss: 0.0034038, lr: 0.0330631\n",
      "[62, 44800/50000] loss: 0.0041574, lr: 0.0330631\n",
      "[62 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[63,  6400/50000] loss: 0.0077219, lr: 0.0315938\n",
      "[63, 12800/50000] loss: 0.0056349, lr: 0.0315938\n",
      "[63, 19200/50000] loss: 0.0057499, lr: 0.0315938\n",
      "[63, 25600/50000] loss: 0.0045835, lr: 0.0315938\n",
      "[63, 32000/50000] loss: 0.0062511, lr: 0.0315938\n",
      "[63, 38400/50000] loss: 0.0060860, lr: 0.0315938\n",
      "[63, 44800/50000] loss: 0.0032513, lr: 0.0315938\n",
      "[63 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[64,  6400/50000] loss: 0.0041128, lr: 0.0301426\n",
      "[64, 12800/50000] loss: 0.0036421, lr: 0.0301426\n",
      "[64, 19200/50000] loss: 0.0038444, lr: 0.0301426\n",
      "[64, 25600/50000] loss: 0.0034874, lr: 0.0301426\n",
      "[64, 32000/50000] loss: 0.0036319, lr: 0.0301426\n",
      "[64, 38400/50000] loss: 0.0035546, lr: 0.0301426\n",
      "[64, 44800/50000] loss: 0.0026877, lr: 0.0301426\n",
      "[64 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[65,  6400/50000] loss: 0.0022596, lr: 0.0287110\n",
      "[65, 12800/50000] loss: 0.0040625, lr: 0.0287110\n",
      "[65, 19200/50000] loss: 0.0040333, lr: 0.0287110\n",
      "[65, 25600/50000] loss: 0.0041225, lr: 0.0287110\n",
      "[65, 32000/50000] loss: 0.0042789, lr: 0.0287110\n",
      "[65, 38400/50000] loss: 0.0023960, lr: 0.0287110\n",
      "[65, 44800/50000] loss: 0.0034659, lr: 0.0287110\n",
      "[65 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[66,  6400/50000] loss: 0.0031155, lr: 0.0273005\n",
      "[66, 12800/50000] loss: 0.0043954, lr: 0.0273005\n",
      "[66, 19200/50000] loss: 0.0037699, lr: 0.0273005\n",
      "[66, 25600/50000] loss: 0.0044987, lr: 0.0273005\n",
      "[66, 32000/50000] loss: 0.0038249, lr: 0.0273005\n",
      "[66, 38400/50000] loss: 0.0038238, lr: 0.0273005\n",
      "[66, 44800/50000] loss: 0.0025101, lr: 0.0273005\n",
      "[66 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[67,  6400/50000] loss: 0.0090867, lr: 0.0259123\n",
      "[67, 12800/50000] loss: 0.0050605, lr: 0.0259123\n",
      "[67, 19200/50000] loss: 0.0068665, lr: 0.0259123\n",
      "[67, 25600/50000] loss: 0.0054048, lr: 0.0259123\n",
      "[67, 32000/50000] loss: 0.0041339, lr: 0.0259123\n",
      "[67, 38400/50000] loss: 0.0050603, lr: 0.0259123\n",
      "[67, 44800/50000] loss: 0.0031721, lr: 0.0259123\n",
      "[67 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[68,  6400/50000] loss: 0.0049529, lr: 0.0245479\n",
      "[68, 12800/50000] loss: 0.0040711, lr: 0.0245479\n",
      "[68, 19200/50000] loss: 0.0033745, lr: 0.0245479\n",
      "[68, 25600/50000] loss: 0.0037603, lr: 0.0245479\n",
      "[68, 32000/50000] loss: 0.0031234, lr: 0.0245479\n",
      "[68, 38400/50000] loss: 0.0033532, lr: 0.0245479\n",
      "[68, 44800/50000] loss: 0.0024202, lr: 0.0245479\n",
      "[68 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[69,  6400/50000] loss: 0.0120189, lr: 0.0232087\n",
      "[69, 12800/50000] loss: 0.0069713, lr: 0.0232087\n",
      "[69, 19200/50000] loss: 0.0045156, lr: 0.0232087\n",
      "[69, 25600/50000] loss: 0.0040760, lr: 0.0232087\n",
      "[69, 32000/50000] loss: 0.0045898, lr: 0.0232087\n",
      "[69, 38400/50000] loss: 0.0037696, lr: 0.0232087\n",
      "[69, 44800/50000] loss: 0.0058418, lr: 0.0232087\n",
      "[69 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[70,  6400/50000] loss: 0.0029934, lr: 0.0218958\n",
      "[70, 12800/50000] loss: 0.0028504, lr: 0.0218958\n",
      "[70, 19200/50000] loss: 0.0041681, lr: 0.0218958\n",
      "[70, 25600/50000] loss: 0.0038163, lr: 0.0218958\n",
      "[70, 32000/50000] loss: 0.0026730, lr: 0.0218958\n",
      "[70, 38400/50000] loss: 0.0047043, lr: 0.0218958\n",
      "[70, 44800/50000] loss: 0.0075574, lr: 0.0218958\n",
      "[70 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[71,  6400/50000] loss: 0.0025912, lr: 0.0206107\n",
      "[71, 12800/50000] loss: 0.0041214, lr: 0.0206107\n",
      "[71, 19200/50000] loss: 0.0031553, lr: 0.0206107\n",
      "[71, 25600/50000] loss: 0.0031512, lr: 0.0206107\n",
      "[71, 32000/50000] loss: 0.0035305, lr: 0.0206107\n",
      "[71, 38400/50000] loss: 0.0028003, lr: 0.0206107\n",
      "[71, 44800/50000] loss: 0.0032894, lr: 0.0206107\n",
      "[71 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[72,  6400/50000] loss: 0.0033513, lr: 0.0193546\n",
      "[72, 12800/50000] loss: 0.0014290, lr: 0.0193546\n",
      "[72, 19200/50000] loss: 0.0021529, lr: 0.0193546\n",
      "[72, 25600/50000] loss: 0.0029885, lr: 0.0193546\n",
      "[72, 32000/50000] loss: 0.0028889, lr: 0.0193546\n",
      "[72, 38400/50000] loss: 0.0033659, lr: 0.0193546\n",
      "[72, 44800/50000] loss: 0.0025378, lr: 0.0193546\n",
      "[72 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[73,  6400/50000] loss: 0.0017552, lr: 0.0181288\n",
      "[73, 12800/50000] loss: 0.0031191, lr: 0.0181288\n",
      "[73, 19200/50000] loss: 0.0018424, lr: 0.0181288\n",
      "[73, 25600/50000] loss: 0.0036790, lr: 0.0181288\n",
      "[73, 32000/50000] loss: 0.0032943, lr: 0.0181288\n",
      "[73, 38400/50000] loss: 0.0030121, lr: 0.0181288\n",
      "[73, 44800/50000] loss: 0.0025232, lr: 0.0181288\n",
      "[73 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[74,  6400/50000] loss: 0.0026002, lr: 0.0169344\n",
      "[74, 12800/50000] loss: 0.0017518, lr: 0.0169344\n",
      "[74, 19200/50000] loss: 0.0016303, lr: 0.0169344\n",
      "[74, 25600/50000] loss: 0.0020134, lr: 0.0169344\n",
      "[74, 32000/50000] loss: 0.0017194, lr: 0.0169344\n",
      "[74, 38400/50000] loss: 0.0021454, lr: 0.0169344\n",
      "[74, 44800/50000] loss: 0.0015061, lr: 0.0169344\n",
      "[74 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[75,  6400/50000] loss: 0.0035414, lr: 0.0157726\n",
      "[75, 12800/50000] loss: 0.0031472, lr: 0.0157726\n",
      "[75, 19200/50000] loss: 0.0019294, lr: 0.0157726\n",
      "[75, 25600/50000] loss: 0.0020172, lr: 0.0157726\n",
      "[75, 32000/50000] loss: 0.0023041, lr: 0.0157726\n",
      "[75, 38400/50000] loss: 0.0019857, lr: 0.0157726\n",
      "[75, 44800/50000] loss: 0.0039878, lr: 0.0157726\n",
      "[75 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[76,  6400/50000] loss: 0.0017162, lr: 0.0146447\n",
      "[76, 12800/50000] loss: 0.0024660, lr: 0.0146447\n",
      "[76, 19200/50000] loss: 0.0024303, lr: 0.0146447\n",
      "[76, 25600/50000] loss: 0.0014864, lr: 0.0146447\n",
      "[76, 32000/50000] loss: 0.0016700, lr: 0.0146447\n",
      "[76, 38400/50000] loss: 0.0020405, lr: 0.0146447\n",
      "[76, 44800/50000] loss: 0.0018220, lr: 0.0146447\n",
      "[76 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[77,  6400/50000] loss: 0.0026645, lr: 0.0135516\n",
      "[77, 12800/50000] loss: 0.0027597, lr: 0.0135516\n",
      "[77, 19200/50000] loss: 0.0017455, lr: 0.0135516\n",
      "[77, 25600/50000] loss: 0.0020429, lr: 0.0135516\n",
      "[77, 32000/50000] loss: 0.0016605, lr: 0.0135516\n",
      "[77, 38400/50000] loss: 0.0014494, lr: 0.0135516\n",
      "[77, 44800/50000] loss: 0.0017419, lr: 0.0135516\n",
      "[77 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[78,  6400/50000] loss: 0.0025340, lr: 0.0124944\n",
      "[78, 12800/50000] loss: 0.0016729, lr: 0.0124944\n",
      "[78, 19200/50000] loss: 0.0013594, lr: 0.0124944\n",
      "[78, 25600/50000] loss: 0.0008658, lr: 0.0124944\n",
      "[78, 32000/50000] loss: 0.0018288, lr: 0.0124944\n",
      "[78, 38400/50000] loss: 0.0024817, lr: 0.0124944\n",
      "[78, 44800/50000] loss: 0.0031578, lr: 0.0124944\n",
      "[78 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[79,  6400/50000] loss: 0.0014331, lr: 0.0114743\n",
      "[79, 12800/50000] loss: 0.0016813, lr: 0.0114743\n",
      "[79, 19200/50000] loss: 0.0014135, lr: 0.0114743\n",
      "[79, 25600/50000] loss: 0.0022254, lr: 0.0114743\n",
      "[79, 32000/50000] loss: 0.0019895, lr: 0.0114743\n",
      "[79, 38400/50000] loss: 0.0020480, lr: 0.0114743\n",
      "[79, 44800/50000] loss: 0.0024915, lr: 0.0114743\n",
      "[79 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[80,  6400/50000] loss: 0.0013203, lr: 0.0104922\n",
      "[80, 12800/50000] loss: 0.0017053, lr: 0.0104922\n",
      "[80, 19200/50000] loss: 0.0020141, lr: 0.0104922\n",
      "[80, 25600/50000] loss: 0.0016296, lr: 0.0104922\n",
      "[80, 32000/50000] loss: 0.0021662, lr: 0.0104922\n",
      "[80, 38400/50000] loss: 0.0013165, lr: 0.0104922\n",
      "[80, 44800/50000] loss: 0.0017996, lr: 0.0104922\n",
      "[80 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[81,  6400/50000] loss: 0.0025050, lr: 0.0095492\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[81, 12800/50000] loss: 0.0024244, lr: 0.0095492\n",
      "[81, 19200/50000] loss: 0.0029955, lr: 0.0095492\n",
      "[81, 25600/50000] loss: 0.0029514, lr: 0.0095492\n",
      "[81, 32000/50000] loss: 0.0020355, lr: 0.0095492\n",
      "[81, 38400/50000] loss: 0.0017162, lr: 0.0095492\n",
      "[81, 44800/50000] loss: 0.0011705, lr: 0.0095492\n",
      "[81 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[82,  6400/50000] loss: 0.0017971, lr: 0.0086460\n",
      "[82, 12800/50000] loss: 0.0017039, lr: 0.0086460\n",
      "[82, 19200/50000] loss: 0.0013340, lr: 0.0086460\n",
      "[82, 25600/50000] loss: 0.0023235, lr: 0.0086460\n",
      "[82, 32000/50000] loss: 0.0034754, lr: 0.0086460\n",
      "[82, 38400/50000] loss: 0.0014668, lr: 0.0086460\n",
      "[82, 44800/50000] loss: 0.0019544, lr: 0.0086460\n",
      "[82 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[83,  6400/50000] loss: 0.0016734, lr: 0.0077836\n",
      "[83, 12800/50000] loss: 0.0012158, lr: 0.0077836\n",
      "[83, 19200/50000] loss: 0.0022578, lr: 0.0077836\n",
      "[83, 25600/50000] loss: 0.0025178, lr: 0.0077836\n",
      "[83, 32000/50000] loss: 0.0011063, lr: 0.0077836\n",
      "[83, 38400/50000] loss: 0.0015819, lr: 0.0077836\n",
      "[83, 44800/50000] loss: 0.0017281, lr: 0.0077836\n",
      "[83 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[84,  6400/50000] loss: 0.0014873, lr: 0.0069629\n",
      "[84, 12800/50000] loss: 0.0026588, lr: 0.0069629\n",
      "[84, 19200/50000] loss: 0.0013394, lr: 0.0069629\n",
      "[84, 25600/50000] loss: 0.0015299, lr: 0.0069629\n",
      "[84, 32000/50000] loss: 0.0014106, lr: 0.0069629\n",
      "[84, 38400/50000] loss: 0.0023160, lr: 0.0069629\n",
      "[84, 44800/50000] loss: 0.0013988, lr: 0.0069629\n",
      "[84 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[85,  6400/50000] loss: 0.0014439, lr: 0.0061847\n",
      "[85, 12800/50000] loss: 0.0017769, lr: 0.0061847\n",
      "[85, 19200/50000] loss: 0.0014495, lr: 0.0061847\n",
      "[85, 25600/50000] loss: 0.0014821, lr: 0.0061847\n",
      "[85, 32000/50000] loss: 0.0016668, lr: 0.0061847\n",
      "[85, 38400/50000] loss: 0.0019279, lr: 0.0061847\n",
      "[85, 44800/50000] loss: 0.0014064, lr: 0.0061847\n",
      "[85 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[86,  6400/50000] loss: 0.0012822, lr: 0.0054497\n",
      "[86, 12800/50000] loss: 0.0022773, lr: 0.0054497\n",
      "[86, 19200/50000] loss: 0.0021908, lr: 0.0054497\n",
      "[86, 25600/50000] loss: 0.0011679, lr: 0.0054497\n",
      "[86, 32000/50000] loss: 0.0013838, lr: 0.0054497\n",
      "[86, 38400/50000] loss: 0.0014219, lr: 0.0054497\n",
      "[86, 44800/50000] loss: 0.0025335, lr: 0.0054497\n",
      "[86 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[87,  6400/50000] loss: 0.0014694, lr: 0.0047586\n",
      "[87, 12800/50000] loss: 0.0011763, lr: 0.0047586\n",
      "[87, 19200/50000] loss: 0.0014057, lr: 0.0047586\n",
      "[87, 25600/50000] loss: 0.0014515, lr: 0.0047586\n",
      "[87, 32000/50000] loss: 0.0016105, lr: 0.0047586\n",
      "[87, 38400/50000] loss: 0.0017934, lr: 0.0047586\n",
      "[87, 44800/50000] loss: 0.0018796, lr: 0.0047586\n",
      "[87 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[88,  6400/50000] loss: 0.0016079, lr: 0.0041123\n",
      "[88, 12800/50000] loss: 0.0016659, lr: 0.0041123\n",
      "[88, 19200/50000] loss: 0.0021805, lr: 0.0041123\n",
      "[88, 25600/50000] loss: 0.0009960, lr: 0.0041123\n",
      "[88, 32000/50000] loss: 0.0015130, lr: 0.0041123\n",
      "[88, 38400/50000] loss: 0.0013046, lr: 0.0041123\n",
      "[88, 44800/50000] loss: 0.0014007, lr: 0.0041123\n",
      "[88 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[89,  6400/50000] loss: 0.0012652, lr: 0.0035112\n",
      "[89, 12800/50000] loss: 0.0023325, lr: 0.0035112\n",
      "[89, 19200/50000] loss: 0.0013599, lr: 0.0035112\n",
      "[89, 25600/50000] loss: 0.0015893, lr: 0.0035112\n",
      "[89, 32000/50000] loss: 0.0016028, lr: 0.0035112\n",
      "[89, 38400/50000] loss: 0.0014428, lr: 0.0035112\n",
      "[89, 44800/50000] loss: 0.0012416, lr: 0.0035112\n",
      "[89 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[90,  6400/50000] loss: 0.0008868, lr: 0.0029560\n",
      "[90, 12800/50000] loss: 0.0021529, lr: 0.0029560\n",
      "[90, 19200/50000] loss: 0.0015731, lr: 0.0029560\n",
      "[90, 25600/50000] loss: 0.0011898, lr: 0.0029560\n",
      "[90, 32000/50000] loss: 0.0013944, lr: 0.0029560\n",
      "[90, 38400/50000] loss: 0.0015069, lr: 0.0029560\n",
      "[90, 44800/50000] loss: 0.0019631, lr: 0.0029560\n",
      "[90 epoch] Accuracy of the network on the validation images: 92 %\n",
      "[91,  6400/50000] loss: 0.0016264, lr: 0.0024472\n",
      "[91, 12800/50000] loss: 0.0017004, lr: 0.0024472\n",
      "[91, 19200/50000] loss: 0.0012460, lr: 0.0024472\n",
      "[91, 25600/50000] loss: 0.0013898, lr: 0.0024472\n",
      "[91, 32000/50000] loss: 0.0015839, lr: 0.0024472\n",
      "[91, 38400/50000] loss: 0.0012634, lr: 0.0024472\n",
      "[91, 44800/50000] loss: 0.0015597, lr: 0.0024472\n",
      "[91 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[92,  6400/50000] loss: 0.0011590, lr: 0.0019853\n",
      "[92, 12800/50000] loss: 0.0011504, lr: 0.0019853\n",
      "[92, 19200/50000] loss: 0.0016639, lr: 0.0019853\n",
      "[92, 25600/50000] loss: 0.0009126, lr: 0.0019853\n",
      "[92, 32000/50000] loss: 0.0012023, lr: 0.0019853\n",
      "[92, 38400/50000] loss: 0.0011704, lr: 0.0019853\n",
      "[92, 44800/50000] loss: 0.0015803, lr: 0.0019853\n",
      "[92 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[93,  6400/50000] loss: 0.0013308, lr: 0.0015708\n",
      "[93, 12800/50000] loss: 0.0019210, lr: 0.0015708\n",
      "[93, 19200/50000] loss: 0.0015186, lr: 0.0015708\n",
      "[93, 25600/50000] loss: 0.0012071, lr: 0.0015708\n",
      "[93, 32000/50000] loss: 0.0012545, lr: 0.0015708\n",
      "[93, 38400/50000] loss: 0.0019587, lr: 0.0015708\n",
      "[93, 44800/50000] loss: 0.0018382, lr: 0.0015708\n",
      "[93 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[94,  6400/50000] loss: 0.0009947, lr: 0.0012042\n",
      "[94, 12800/50000] loss: 0.0012183, lr: 0.0012042\n",
      "[94, 19200/50000] loss: 0.0010225, lr: 0.0012042\n",
      "[94, 25600/50000] loss: 0.0019140, lr: 0.0012042\n",
      "[94, 32000/50000] loss: 0.0018152, lr: 0.0012042\n",
      "[94, 38400/50000] loss: 0.0014633, lr: 0.0012042\n",
      "[94, 44800/50000] loss: 0.0014259, lr: 0.0012042\n",
      "[94 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[95,  6400/50000] loss: 0.0015569, lr: 0.0008856\n",
      "[95, 12800/50000] loss: 0.0016193, lr: 0.0008856\n",
      "[95, 19200/50000] loss: 0.0014506, lr: 0.0008856\n",
      "[95, 25600/50000] loss: 0.0015207, lr: 0.0008856\n",
      "[95, 32000/50000] loss: 0.0007044, lr: 0.0008856\n",
      "[95, 38400/50000] loss: 0.0015179, lr: 0.0008856\n",
      "[95, 44800/50000] loss: 0.0012098, lr: 0.0008856\n",
      "[95 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[96,  6400/50000] loss: 0.0016940, lr: 0.0006156\n",
      "[96, 12800/50000] loss: 0.0013985, lr: 0.0006156\n",
      "[96, 19200/50000] loss: 0.0018002, lr: 0.0006156\n",
      "[96, 25600/50000] loss: 0.0021061, lr: 0.0006156\n",
      "[96, 32000/50000] loss: 0.0014888, lr: 0.0006156\n",
      "[96, 38400/50000] loss: 0.0018621, lr: 0.0006156\n",
      "[96, 44800/50000] loss: 0.0015038, lr: 0.0006156\n",
      "[96 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[97,  6400/50000] loss: 0.0016202, lr: 0.0003943\n",
      "[97, 12800/50000] loss: 0.0015803, lr: 0.0003943\n",
      "[97, 19200/50000] loss: 0.0008444, lr: 0.0003943\n",
      "[97, 25600/50000] loss: 0.0018910, lr: 0.0003943\n",
      "[97, 32000/50000] loss: 0.0008008, lr: 0.0003943\n",
      "[97, 38400/50000] loss: 0.0008720, lr: 0.0003943\n",
      "[97, 44800/50000] loss: 0.0013352, lr: 0.0003943\n",
      "[97 epoch] Accuracy of the network on the validation images: 92 %\n",
      "[98,  6400/50000] loss: 0.0014908, lr: 0.0002219\n",
      "[98, 12800/50000] loss: 0.0008606, lr: 0.0002219\n",
      "[98, 19200/50000] loss: 0.0012299, lr: 0.0002219\n",
      "[98, 25600/50000] loss: 0.0021738, lr: 0.0002219\n",
      "[98, 32000/50000] loss: 0.0013824, lr: 0.0002219\n",
      "[98, 38400/50000] loss: 0.0012567, lr: 0.0002219\n",
      "[98, 44800/50000] loss: 0.0016558, lr: 0.0002219\n",
      "[98 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[99,  6400/50000] loss: 0.0011962, lr: 0.0000987\n",
      "[99, 12800/50000] loss: 0.0014694, lr: 0.0000987\n",
      "[99, 19200/50000] loss: 0.0018999, lr: 0.0000987\n",
      "[99, 25600/50000] loss: 0.0018975, lr: 0.0000987\n",
      "[99, 32000/50000] loss: 0.0011674, lr: 0.0000987\n",
      "[99, 38400/50000] loss: 0.0009556, lr: 0.0000987\n",
      "[99, 44800/50000] loss: 0.0015590, lr: 0.0000987\n",
      "[99 epoch] Accuracy of the network on the validation images: 91 %\n",
      "[100,  6400/50000] loss: 0.0017926, lr: 0.0000247\n",
      "[100, 12800/50000] loss: 0.0013264, lr: 0.0000247\n",
      "[100, 19200/50000] loss: 0.0007775, lr: 0.0000247\n",
      "[100, 25600/50000] loss: 0.0015916, lr: 0.0000247\n",
      "[100, 32000/50000] loss: 0.0009884, lr: 0.0000247\n",
      "[100, 38400/50000] loss: 0.0015495, lr: 0.0000247\n",
      "[100, 44800/50000] loss: 0.0015448, lr: 0.0000247\n",
      "[100 epoch] Accuracy of the network on the validation images: 91 %\n",
      "Finished Training\n"
     ]
    }
   ],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr=initial_lr, momentum=0.9)\n",
    "learning_rate_scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=num_epoch)\n",
    "\n",
    "for epoch in range(num_epoch):  \n",
    "    learning_rate_scheduler.step()\n",
    "    running_loss = 0.0\n",
    "    \n",
    "    for i, data in enumerate(train_loader, 0):\n",
    "        inputs, labels = data\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        \n",
    "        optimizer.step()\n",
    "        \n",
    "        running_loss += loss.item()\n",
    "                \n",
    "        show_period = 100\n",
    "        if i % show_period ==  show_period-1:    # print every \"show_period\" mini-batches\n",
    "            print('[%d, %5d/50000] loss: %.7f, lr: %.7f' %\n",
    "                  (epoch + 1, (i + 1)*batch_size, running_loss / show_period, learning_rate_scheduler.get_lr()[0]))\n",
    "            running_loss = 0.0\n",
    "            \n",
    "    # validation part\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for i, data in enumerate(valid_loader, 0):\n",
    "        inputs, labels = data\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "        outputs = net(inputs)\n",
    "        \n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total += labels.size(0)\n",
    "        correct += (predicted == labels).sum().item()\n",
    "        \n",
    "    print('[%d epoch] Accuracy of the network on the validation images: %d %%' % \n",
    "          (epoch + 1, 100 * correct / total)\n",
    "         )\n",
    "\n",
    "print('Finished Training')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "y3pEm3QGFw7m"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of the network on the 10000 test images: 90 %\n",
      "Accuracy of plane : 92 %\n",
      "Accuracy of   car : 95 %\n",
      "Accuracy of  bird : 87 %\n",
      "Accuracy of   cat : 81 %\n",
      "Accuracy of  deer : 92 %\n",
      "Accuracy of   dog : 84 %\n",
      "Accuracy of  frog : 93 %\n",
      "Accuracy of horse : 93 %\n",
      "Accuracy of  ship : 95 %\n",
      "Accuracy of truck : 93 %\n"
     ]
    }
   ],
   "source": [
    "class_correct = list(0. for i in range(10))\n",
    "class_total = list(0. for i in range(10))\n",
    "\n",
    "correct = 0\n",
    "total = 0\n",
    "\n",
    "with torch.no_grad():\n",
    "    for data in test_loader:\n",
    "        images, labels = data\n",
    "        images, labels = images.to(device), labels.to(device)\n",
    "        outputs = net(images)\n",
    "        _, predicted = torch.max(outputs, 1)\n",
    "        c = (predicted == labels).squeeze()\n",
    "                \n",
    "        for i in range(labels.shape[0]):\n",
    "            label = labels[i]\n",
    "            class_correct[label] += c[i].item()\n",
    "            class_total[label] += 1\n",
    "            \n",
    "            total += labels.size(0)\n",
    "            correct += (predicted == labels).sum().item()\n",
    "\n",
    "print('Accuracy of the network on the 10000 test images: %d %%' % (\n",
    "    100 * correct / total))            \n",
    "            \n",
    "for i in range(10):\n",
    "    print('Accuracy of %5s : %2d %%' % (\n",
    "        classes[i], 100 * class_correct[i] / class_total[i])) "
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "PeleeNet-PyTorch.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "pytorch_1.0",
   "language": "python",
   "name": "pytorch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}