hw5_Q1_1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "streaming-midwest",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/python\n",
    "# -*- coding: utf-8 -*-\n",
    "\"\"\"\n",
    "@author: Xiao Jin\n",
    "In this part we write the hw5\n",
    "\"\"\"\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from plot import visual, plot\n",
    "\n",
    "# 32x32x3\n",
    "(training_data, training_label), (testing_data, testing_label) = tf.keras.datasets.cifar10.load_data()\n",
    "N_train = training_data.shape[0]\n",
    "N_test = testing_data.shape[0]\n",
    "step_size = 0.001\n",
    "training_epochs = 40\n",
    "batch_size = 500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "requested-alignment",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = (\n",
    "    tf.data.Dataset.from_tensor_slices((training_data, training_label)).batch(\n",
    "        batch_size).shuffle(buffer_size=N_train, seed=0)\n",
    ")\n",
    "\n",
    "\n",
    "train_dataset = (train_dataset.map(lambda x, y:\n",
    "                                   (tf.divide(tf.cast(x, tf.float32), 255.0),\n",
    "                                    tf.reshape(tf.one_hot(y, 10), (-1, 10))))\n",
    ")\n",
    "\n",
    "# data augmentation\n",
    "train_dataset = (train_dataset.map(lambda x, y:\n",
    "                                   (tf.image.random_flip_left_right(x),y)\n",
    "))\n",
    "\n",
    "train_dataset = (train_dataset.map(lambda x, y:\n",
    "                                   (tf.image.random_contrast(x, lower=0.0, upper=1.0),y)\n",
    "))\n",
    "\n",
    "test_dataset = (\n",
    "    tf.data.Dataset.from_tensor_slices((testing_data, testing_label)).batch(N_test)\n",
    ")\n",
    "\n",
    "test_dataset = (\n",
    "    test_dataset.map(lambda x, y:\n",
    "                      (tf.divide(tf.cast(x, tf.float32), 255.0),\n",
    "                       tf.reshape(tf.one_hot(y, 10), (-1, 10))))\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "specialized-organ",
   "metadata": {},
   "outputs": [],
   "source": [
    "class neural_netowrk(tf.keras.Model):\n",
    "    def __init__(self, seed=1):\n",
    "        super(neural_netowrk, self).__init__()\n",
    "        # use random seed to make the initialization repeat\n",
    "        tf.random.set_seed(seed)\n",
    "        # define convolutional layers\n",
    "        self.c1 = tf.keras.layers.Conv2D(32, kernel_size=5, activation = 'relu', name='c1')\n",
    "        self.m1 = tf.keras.layers.MaxPool2D(pool_size=(2,2), strides=2, name='m1')\n",
    "        self.c2 = tf.keras.layers.Conv2D(32, kernel_size=5, activation = 'relu', name='c2')\n",
    "        self.m2 = tf.keras.layers.MaxPool2D(pool_size=(2,2), strides=2, name='m2')\n",
    "        self.c3 = tf.keras.layers.Conv2D(64, kernel_size=3, activation='relu', name='c3')\n",
    "        self.fc1 = tf.keras.layers.Dense(10)\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        here we define the forward function\n",
    "        :param input: the input data\n",
    "        :return: output tensor\n",
    "        '''\n",
    "        # For each layer, a bias will also be initialized and add to the output after matrix multiply.\n",
    "        x = self.c1(input)\n",
    "        x = self.m1(x)\n",
    "        x = self.c2(x)\n",
    "        x = self.m2(x)\n",
    "        x = self.c3(x)\n",
    "        x = tf.keras.layers.Flatten()(x)\n",
    "        output = self.fc1(x)\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "valuable-degree",
   "metadata": {},
   "outputs": [],
   "source": [
    "class input_block(tf.keras.Model):\n",
    "    def __init__(self, seed=1):\n",
    "        super(input_block, self).__init__()\n",
    "        # use random seed to make the initialization repeat\n",
    "        tf.random.set_seed(seed)\n",
    "        # define convolutional layers\n",
    "        self.c1 = tf.keras.layers.Conv2D(64, kernel_size=7, strides=2, padding='VALID', activation = None, name='c1')\n",
    "        self.bn1 = tf.keras.layers.BatchNormalization()\n",
    "        self.relu = tf.keras.activations.relu\n",
    "        self.maxpool = tf.keras.layers.MaxPool2D(3, strides=2, padding='VALID')\n",
    "\n",
    "    def forward(self, input):\n",
    "        # batch_size x frame x 224 x 224 x 3\n",
    "        x = self.c1(input)\n",
    "        # batch_size x frame x 112 x 112 x 64\n",
    "        x = self.bn1(x)\n",
    "        x = self.relu(x)\n",
    "        output = self.maxpool(x)\n",
    "        # batch_size x frame x 56 x 56 x 64\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "written-symbol",
   "metadata": {},
   "outputs": [],
   "source": [
    "class basic_block(tf.keras.Model):\n",
    "    def __init__(self, planes, stride=1, seed=1):\n",
    "        super(basic_block, self).__init__()\n",
    "        # use random seed to make the initialization repeat\n",
    "        tf.random.set_seed(seed)\n",
    "        # define convolutional layers\n",
    "        self.conv1 = tf.keras.layers.Conv2D(planes, kernel_size=3, strides=stride, padding='SAME', activation = None)\n",
    "        self.bn1 = tf.keras.layers.BatchNormalization()\n",
    "        self.conv2 = tf.keras.layers.Conv2D(planes, kernel_size=3, padding='SAME', activation=None)\n",
    "        self.bn2 = tf.keras.layers.BatchNormalization()\n",
    "        self.relu = tf.keras.activations.relu\n",
    "        self.stride = stride\n",
    "        if self.stride != 1:\n",
    "            self.short_cut = tf.keras.layers.Conv2D(planes, kernel_size=1, strides=stride, padding='SAME',\n",
    "                                                    activation = None)\n",
    "            self.bn_short = tf.keras.layers.BatchNormalization()\n",
    "\n",
    "    def forward(self, input):\n",
    "        # batch_size x frame x H x W x planes\n",
    "        x_1 = self.conv1(input)\n",
    "        x_1 = self.bn1(x_1)\n",
    "        x_1 = self.relu(x_1)\n",
    "        x_1 = self.conv2(x_1)\n",
    "        x_1 = self.bn2(x_1)\n",
    "        if self.stride != 1:\n",
    "            # shot cut\n",
    "            x_2 = self.short_cut(input)\n",
    "            x_2 = self.bn_short(x_2)\n",
    "            # batch_size x frame x H/2 x W/2 x planes\n",
    "            output = self.relu(x_1 + x_2)\n",
    "        else:\n",
    "            output = self.relu(x_1 + input)\n",
    "            # batch_size x frame x H x W x planes\n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "incorporate-string",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ResNet18(tf.keras.Model):\n",
    "    def __init__(self):\n",
    "        super(ResNet18, self).__init__()\n",
    "        # use random seed to make the initialization repeat\n",
    "        # CNN 1-2\n",
    "        self.input_part = input_block()\n",
    "        # CNN 3-6\n",
    "        self.block_11 = basic_block(planes=64, seed=1)\n",
    "        self.block_12 = basic_block(planes=64, seed=2)\n",
    "        # CNN 6-10\n",
    "        self.block_21 = basic_block(planes=128, stride=2, seed=1)\n",
    "        self.block_22 = basic_block(planes=128, seed=2)\n",
    "        # CNN 10-14\n",
    "        self.block_31 = basic_block(planes=256, stride=2, seed=1)\n",
    "        self.block_32 = basic_block(planes=256, seed=2)\n",
    "        # CNN 14-18\n",
    "        self.block_41 = basic_block(planes=512, stride=2, seed=1)\n",
    "        self.block_42 = basic_block(planes=512, seed=2)\n",
    "        # Avg pooling\n",
    "        self.Avg = tf.keras.layers.AvgPool2D(7, padding='VALID')\n",
    "        self.p1 = tf.keras.layers.Dense(10, name='p1')\n",
    "        self.relu = tf.keras.activations.relu\n",
    "        self.dropout = tf.keras.layers.Dropout(0.5)\n",
    "\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        here we define the forward function\n",
    "        :param input: the input data\n",
    "        :return: output tensor\n",
    "        '''\n",
    "        # For each layer, a bias will also be initialized and add to the output after matrix multiply.\n",
    "        # reshape\n",
    "        # image interpolation\n",
    "        x = tf.image.resize(input, [-1, 224, 224, 3])\n",
    "        # batch size x 224 x 224 x 3\n",
    "        x = self.input_part.forward(x)\n",
    "        # batch size x 56 x 56 x 64\n",
    "        x = self.block_11.forward(x)\n",
    "        x = self.block_12.forward(x)\n",
    "        # batch size x 56 x 56 x 64\n",
    "        x = self.block_21.forward(x)\n",
    "        x = self.block_22.forward(x)\n",
    "        # batch size x 28 x 28 x 128\n",
    "        x = self.block_31.forward(x)\n",
    "        x = self.block_32.forward(x)\n",
    "        # batch size x 14 x 14 x 256\n",
    "        x = self.block_41.forward(x)\n",
    "        x = self.block_42.forward(x)\n",
    "        # batch size x 7 x 7 x 512\n",
    "        x = self.Avg(x)\n",
    "        # batch size x 1 x 1 x 512\n",
    "        x = tf.reshape(x, [-1, 512])\n",
    "        output = self.p1(x)\n",
    "        \n",
    "        return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "incident-gather",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_accuracy(predict, labels):\n",
    "    '''\n",
    "    Compute accuracy\n",
    "    :param predict: predict vector\n",
    "    :param labels: ground truth\n",
    "    :return: accuracy, accuracy for each digit\n",
    "    '''\n",
    "    # predict vector\n",
    "    predict = tf.one_hot(tf.math.argmax(predict, 1), 10, dtype=tf.float32)\n",
    "    # accuracy\n",
    "    avg_accuracy = tf.reduce_mean(tf.reduce_sum(tf.math.multiply(labels, predict), axis=1))\n",
    "    return avg_accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "acute-adjustment",
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, data):\n",
    "    test_x, test_y = zip(*data)\n",
    "    test_x = test_x[0]\n",
    "    test_y= test_y[0]\n",
    "    output = model.forward(test_x)\n",
    "    test_acc = compute_accuracy(output, test_y)\n",
    "    avg_error = 1 - test_acc\n",
    "    # classification error for each digit\n",
    "    error_for_each_class = []\n",
    "    predict = tf.one_hot(tf.math.argmax(output, 1), 10, dtype=tf.float32)\n",
    "    for each_class in range(test_y.numpy().shape[1]):\n",
    "        num_correct = tf.reduce_sum(tf.math.multiply(predict[:, each_class], test_y[:, each_class]))\n",
    "        num_labels = tf.reduce_sum(test_y[:, each_class])\n",
    "        error_for_each_class.append(1 - num_correct / num_labels)\n",
    "    return avg_error.numpy(), error_for_each_class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "favorite-editing",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_loss(predict, labels):\n",
    "    return tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels, predict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "biblical-volunteer",
   "metadata": {},
   "outputs": [],
   "source": [
    "def check(model, train_dataset, test_dataset):\n",
    "    # we compute the training loss & accuracy / testing loss & accuracy\n",
    "    '''\n",
    "    :param model: the neural network\n",
    "    :return: training loss; training accuracy; testing loss; testing accuracy\n",
    "    '''\n",
    "    loss = []\n",
    "    acc = []\n",
    "    for x, y in train_dataset:\n",
    "        output = model.forward(x)\n",
    "        loss.append(compute_loss(output, y))\n",
    "        acc.append(compute_accuracy(output, y))\n",
    "    train_loss = sum(loss) / len(loss)\n",
    "    train_acc = sum(acc) / len(acc)\n",
    "    test_x, test_y = zip(*test_dataset)\n",
    "    test_x = test_x[0]\n",
    "    test_y = test_y[0]\n",
    "    output = model.forward(test_x)\n",
    "    test_loss = compute_loss(output, test_y)\n",
    "    test_acc = compute_accuracy(output, test_y)\n",
    "    return train_loss.numpy(), train_acc.numpy(), test_loss.numpy(), test_acc.numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "id": "floating-dublin",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(train_data, test_data):\n",
    "    epoch = 0\n",
    "    model = neural_netowrk()\n",
    "    # model_2 = ResNet18()\n",
    "    optimizer = tf.keras.optimizers.Adam(learning_rate=step_size)\n",
    "    training_loss = []\n",
    "    training_acc = []\n",
    "    testing_loss = []\n",
    "    testing_acc = []\n",
    "    while epoch < training_epochs:\n",
    "        epoch += 1\n",
    "        for x, y in train_data:\n",
    "            with tf.GradientTape() as tape:\n",
    "                predict = model.forward(x)\n",
    "                loss = compute_loss(predict, y)\n",
    "            gradients = tape.gradient(loss, model.trainable_variables)\n",
    "            optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "        train_loss, train_acc, test_loss, test_acc = check(model, train_data, test_data)\n",
    "        training_loss.append(train_loss)\n",
    "        training_acc.append(train_acc)\n",
    "        testing_loss.append(test_loss)\n",
    "        testing_acc.append(test_acc)\n",
    "        print('Epoch:', epoch, 'training loss:', training_loss[-1], 'training accuracy', training_acc[-1],\n",
    "              'testing loss', testing_loss[-1], 'testing accuracy', testing_acc[-1])\n",
    "    # visualize filters\n",
    "    # visual(model.trainable_variables[0])\n",
    "    # plot\n",
    "    plot(training_loss, training_acc, testing_loss, testing_acc)\n",
    "    # save model\n",
    "    model.save_weights('model_weights.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "threatened-isolation",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1 training loss: 1.9727385 training accuracy 0.27247998 testing loss 1.9210447 testing accuracy 0.3075\n",
      "Epoch: 2 training loss: 1.8596867 training accuracy 0.33318004 testing loss 1.7962141 testing accuracy 0.354\n",
      "Epoch: 3 training loss: 1.7702119 training accuracy 0.3587599 testing loss 1.6718881 testing accuracy 0.4078\n",
      "Epoch: 4 training loss: 1.6700692 training accuracy 0.39138004 testing loss 1.5665354 testing accuracy 0.4435\n",
      "Epoch: 5 training loss: 1.6282938 training accuracy 0.41743988 testing loss 1.5786536 testing accuracy 0.4474\n",
      "Epoch: 6 training loss: 1.5766126 training accuracy 0.43394005 testing loss 1.4858078 testing accuracy 0.4739\n",
      "Epoch: 7 training loss: 1.5569609 training accuracy 0.44179988 testing loss 1.5128695 testing accuracy 0.4752\n",
      "Epoch: 8 training loss: 1.4945571 training accuracy 0.46762002 testing loss 1.4314053 testing accuracy 0.4996\n",
      "Epoch: 9 training loss: 1.5050185 training accuracy 0.45749995 testing loss 1.4150532 testing accuracy 0.5025\n",
      "Epoch: 10 training loss: 1.4598721 training accuracy 0.48226002 testing loss 1.3820456 testing accuracy 0.5178\n",
      "Epoch: 11 training loss: 1.43155 training accuracy 0.48880008 testing loss 1.3569244 testing accuracy 0.5273\n",
      "Epoch: 12 training loss: 1.4237356 training accuracy 0.50020015 testing loss 1.3425006 testing accuracy 0.5358\n",
      "Epoch: 13 training loss: 1.3745341 training accuracy 0.51348 testing loss 1.295412 testing accuracy 0.5466\n",
      "Epoch: 14 training loss: 1.3760556 training accuracy 0.5096199 testing loss 1.3030808 testing accuracy 0.5467\n",
      "Epoch: 15 training loss: 1.4098558 training accuracy 0.50308 testing loss 1.311584 testing accuracy 0.5502\n",
      "Epoch: 16 training loss: 1.3280514 training accuracy 0.53470004 testing loss 1.311332 testing accuracy 0.5497\n",
      "Epoch: 17 training loss: 1.3137102 training accuracy 0.5382601 testing loss 1.2584033 testing accuracy 0.5675\n",
      "Epoch: 18 training loss: 1.3487707 training accuracy 0.5167599 testing loss 1.2857734 testing accuracy 0.5585\n",
      "Epoch: 19 training loss: 1.3475139 training accuracy 0.52886003 testing loss 1.282558 testing accuracy 0.5606\n",
      "Epoch: 20 training loss: 1.3004918 training accuracy 0.5416601 testing loss 1.2374537 testing accuracy 0.5759\n",
      "Epoch: 21 training loss: 1.2901666 training accuracy 0.5457001 testing loss 1.2222542 testing accuracy 0.5748\n",
      "Epoch: 22 training loss: 1.2655364 training accuracy 0.5566599 testing loss 1.1911547 testing accuracy 0.5876\n",
      "Epoch: 23 training loss: 1.2149258 training accuracy 0.57328004 testing loss 1.1869353 testing accuracy 0.5941\n",
      "Epoch: 24 training loss: 1.2208768 training accuracy 0.57537985 testing loss 1.1730748 testing accuracy 0.5987\n",
      "Epoch: 25 training loss: 1.257374 training accuracy 0.55494 testing loss 1.1845416 testing accuracy 0.5942\n",
      "Epoch: 26 training loss: 1.2234758 training accuracy 0.5713401 testing loss 1.1508628 testing accuracy 0.6032\n",
      "Epoch: 27 training loss: 1.2433264 training accuracy 0.56171995 testing loss 1.1699793 testing accuracy 0.6029\n",
      "Epoch: 28 training loss: 1.2100388 training accuracy 0.5765001 testing loss 1.1554731 testing accuracy 0.5991\n",
      "Epoch: 29 training loss: 1.1812255 training accuracy 0.58824015 testing loss 1.141504 testing accuracy 0.6086\n",
      "Epoch: 30 training loss: 1.1598868 training accuracy 0.5963397 testing loss 1.151771 testing accuracy 0.6064\n",
      "Epoch: 31 training loss: 1.290003 training accuracy 0.55086 testing loss 1.1693443 testing accuracy 0.589\n",
      "Epoch: 32 training loss: 1.1585736 training accuracy 0.59630007 testing loss 1.1235316 testing accuracy 0.6116\n",
      "Epoch: 33 training loss: 1.1310574 training accuracy 0.6106199 testing loss 1.1218513 testing accuracy 0.617\n",
      "Epoch: 34 training loss: 1.1606245 training accuracy 0.58754003 testing loss 1.1217418 testing accuracy 0.6164\n",
      "Epoch: 35 training loss: 1.153202 training accuracy 0.59511995 testing loss 1.1161875 testing accuracy 0.6159\n",
      "Epoch: 36 training loss: 1.1655079 training accuracy 0.59366 testing loss 1.1184393 testing accuracy 0.6177\n",
      "Epoch: 37 training loss: 1.1378825 training accuracy 0.6030401 testing loss 1.0925615 testing accuracy 0.6249\n",
      "Epoch: 38 training loss: 1.1461463 training accuracy 0.6008599 testing loss 1.1082537 testing accuracy 0.6172\n",
      "Epoch: 39 training loss: 1.1432949 training accuracy 0.6025601 testing loss 1.1158931 testing accuracy 0.6272\n",
      "Epoch: 40 training loss: 1.0790825 training accuracy 0.6247401 testing loss 1.0914824 testing accuracy 0.6274\n"
     ]
    }
   ],
   "source": [
    "train(train_dataset, test_dataset)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "laden-graph",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}