18BCE080_DL_PRAC6.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "18BCE080_DL_PRAC6.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kd6sWfeKXQHF"
      },
      "source": [
        "# Classification on MNIST using Stacked Autoencoders "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jroNCWsHXSoY"
      },
      "source": [
        "## Importing necessary libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d8fNCDn2Wod0"
      },
      "source": [
        "import keras\n",
        "from keras.datasets import mnist\n",
        "from keras.models import Sequential,Model\n",
        "from keras.layers import Dense,Input\n",
        "from keras.utils import to_categorical\n",
        "\n",
        "import numpy as np\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qhWyqafPX0pX"
      },
      "source": [
        "## Loading Dataset and preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IX5x-xoHXPAy"
      },
      "source": [
        "# Loading data\n",
        "(X_train,y_train), (X_test,y_test) = mnist.load_data()\n",
        "\n",
        "# Flattening the data\n",
        "X_train = np.reshape(X_train,(60000,784)).astype('float32')\n",
        "X_test = np.reshape(X_test,(10000,784)).astype('float32')\n",
        "\n",
        "# Converting the target data to one - hot encoding\n",
        "y_train = to_categorical(y_train,num_classes=10)\n",
        "y_test = to_categorical(y_test,num_classes=10)\n",
        "\n",
        "# Normalizing the data\n",
        "X_train = X_train/255\n",
        "X_test = X_test/255"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kp4FCrA-aYWB"
      },
      "source": [
        "## Autoencoder 1\n",
        "\n",
        "784 -> 100 -> 784"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ma_k3-SWmh32"
      },
      "source": [
        "**Declaration and training**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dWm6E3mOYvYF",
        "outputId": "d228743e-6aa5-4462-eb99-98ebfdda8a15"
      },
      "source": [
        "input_main = Input(shape=(784,))\n",
        "h1 = Dense(100,activation='sigmoid')(input_main)\n",
        "o1 = Dense(784,activation='sigmoid')(h1)\n",
        "\n",
        "ae1 = Model(inputs=input_main,outputs=o1)\n",
        "\n",
        "ae1.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_6\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_7 (InputLayer)         [(None, 784)]             0         \n",
            "_________________________________________________________________\n",
            "dense_22 (Dense)             (None, 100)               78500     \n",
            "_________________________________________________________________\n",
            "dense_23 (Dense)             (None, 784)               79184     \n",
            "=================================================================\n",
            "Total params: 157,684\n",
            "Trainable params: 157,684\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aHg91xELZtsC",
        "outputId": "a414009b-9c4d-4d14-ba5e-020913c02200"
      },
      "source": [
        "ae1.compile(optimizer='adam',loss='mse',metrics=['accuracy'])\n",
        "ae1.fit(X_train,X_train,epochs=10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.0601 - accuracy: 0.0122\n",
            "Epoch 2/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.0214 - accuracy: 0.0101\n",
            "Epoch 3/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.0132 - accuracy: 0.0116\n",
            "Epoch 4/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.0098 - accuracy: 0.0116\n",
            "Epoch 5/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.0081 - accuracy: 0.0124\n",
            "Epoch 6/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.0068 - accuracy: 0.0133\n",
            "Epoch 7/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.0057 - accuracy: 0.0148\n",
            "Epoch 8/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.0048 - accuracy: 0.0142\n",
            "Epoch 9/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.0041 - accuracy: 0.0141\n",
            "Epoch 10/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.0036 - accuracy: 0.0150\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7fe648320610>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 56
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0vUR71p7mobq"
      },
      "source": [
        "**Taking output of hidden layer as input to Autoencoder 2**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EnbYOdP7au7t",
        "outputId": "2d229306-56f7-415e-d3f8-458ebf7134fb"
      },
      "source": [
        "ae1.layers[1].output"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<KerasTensor: shape=(None, 100) dtype=float32 (created by layer 'dense_22')>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "q8TGX32dbqxJ"
      },
      "source": [
        "trimmed_ae1 = Model(inputs=input_main,outputs=ae1.layers[1].output)\n",
        "\n",
        "X_train_ae2 = trimmed_ae1.predict(X_train)\n",
        "X_test_ae2 = trimmed_ae1.predict(X_test)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qVEmoU7RmvFV"
      },
      "source": [
        "## Autoencoder 2\n",
        "\n",
        "100 -> 50 -> 100"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nZ_Sbdrpm1jU"
      },
      "source": [
        "**Declaration and training**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-8ZDTeyPaGMP",
        "outputId": "4ae6ec01-c0d0-4775-a92d-b3bdbb1ace09"
      },
      "source": [
        "input_ae2 = Input(shape=(100,))\n",
        "h2 = Dense(50,activation='sigmoid')(input_ae2)\n",
        "o2 = Dense(100,activation='sigmoid')(h2)\n",
        "\n",
        "ae2 = Model(inputs=input_ae2,outputs=o2)\n",
        "\n",
        "ae2.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_8\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_8 (InputLayer)         [(None, 100)]             0         \n",
            "_________________________________________________________________\n",
            "dense_24 (Dense)             (None, 50)                5050      \n",
            "_________________________________________________________________\n",
            "dense_25 (Dense)             (None, 100)               5100      \n",
            "=================================================================\n",
            "Total params: 10,150\n",
            "Trainable params: 10,150\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZqC5qatTfU94",
        "outputId": "a00d058b-f8fc-4a97-b2d6-7df33c0c0cca"
      },
      "source": [
        "ae2.compile(optimizer='adam',loss='mse',metrics=['accuracy'])\n",
        "ae2.fit(X_train_ae2,X_train_ae2,epochs=10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "1875/1875 [==============================] - 3s 1ms/step - loss: 0.0651 - accuracy: 0.0279\n",
            "Epoch 2/10\n",
            "1875/1875 [==============================] - 3s 1ms/step - loss: 0.0316 - accuracy: 0.0759\n",
            "Epoch 3/10\n",
            "1875/1875 [==============================] - 3s 1ms/step - loss: 0.0244 - accuracy: 0.1165\n",
            "Epoch 4/10\n",
            "1875/1875 [==============================] - 3s 1ms/step - loss: 0.0228 - accuracy: 0.1294\n",
            "Epoch 5/10\n",
            "1875/1875 [==============================] - 3s 1ms/step - loss: 0.0220 - accuracy: 0.1387\n",
            "Epoch 6/10\n",
            "1875/1875 [==============================] - 3s 1ms/step - loss: 0.0215 - accuracy: 0.1394\n",
            "Epoch 7/10\n",
            "1875/1875 [==============================] - 3s 1ms/step - loss: 0.0211 - accuracy: 0.1455\n",
            "Epoch 8/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.0209 - accuracy: 0.1451\n",
            "Epoch 9/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.0206 - accuracy: 0.1473\n",
            "Epoch 10/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.0205 - accuracy: 0.1482\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7fe648c41610>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N3ygjyQrm4-g"
      },
      "source": [
        "**Taking output of hidden layer as input to classifier**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A2y1eUOCifuJ"
      },
      "source": [
        "trimmed_ae2 = Model(inputs=input_ae2,outputs=ae2.layers[1].output)\n",
        "\n",
        "X_train_clf = trimmed_ae2.predict(X_train_ae2)\n",
        "X_test_clf = trimmed_ae2.predict(X_test_ae2)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5FXW5iPsm8Mn"
      },
      "source": [
        "## Classifier\n",
        "\n",
        "50 -> 10"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fjp5iZGfnBkC"
      },
      "source": [
        "**Declaration and training**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MXCDYYUnjWo-",
        "outputId": "fde3dc6c-a006-449f-fd9b-6e235cfed862"
      },
      "source": [
        "input_clf = Input((50,))\n",
        "f_output = Dense(10,activation='softmax')(input_clf)\n",
        "\n",
        "clf = Model(inputs = input_clf, outputs = (f_output))\n",
        "\n",
        "clf.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_10\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "input_9 (InputLayer)         [(None, 50)]              0         \n",
            "_________________________________________________________________\n",
            "dense_26 (Dense)             (None, 10)                510       \n",
            "=================================================================\n",
            "Total params: 510\n",
            "Trainable params: 510\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2bDPa-Imj0GQ",
        "outputId": "06b50593-42e1-4ddd-b88c-691f841e85da"
      },
      "source": [
        "clf.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'])\n",
        "clf.fit(X_train_clf,y_train,epochs=10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 2.0090 - accuracy: 0.3948\n",
            "Epoch 2/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 1.0771 - accuracy: 0.8132\n",
            "Epoch 3/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.7719 - accuracy: 0.8361\n",
            "Epoch 4/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.6395 - accuracy: 0.8494\n",
            "Epoch 5/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.5675 - accuracy: 0.8569\n",
            "Epoch 6/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.5268 - accuracy: 0.8628\n",
            "Epoch 7/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.4917 - accuracy: 0.8670\n",
            "Epoch 8/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.4746 - accuracy: 0.8690\n",
            "Epoch 9/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.4582 - accuracy: 0.8721\n",
            "Epoch 10/10\n",
            "1875/1875 [==============================] - 3s 2ms/step - loss: 0.4493 - accuracy: 0.8733\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7fe648310b90>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 64
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OMt2rZwdnEnp"
      },
      "source": [
        "## Fine-tuned Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ez5COmlekIOo",
        "outputId": "c7e08103-54ec-4a57-8110-26c009410c8b"
      },
      "source": [
        "model = Sequential()\n",
        "model.add(ae1.layers[0])\n",
        "model.add(ae1.layers[1])\n",
        "model.add(ae2.layers[1])\n",
        "model.add(clf.layers[-1])\n",
        "\n",
        "model.summary()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential_13\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "dense_22 (Dense)             (None, 100)               78500     \n",
            "_________________________________________________________________\n",
            "dense_24 (Dense)             (None, 50)                5050      \n",
            "_________________________________________________________________\n",
            "dense_26 (Dense)             (None, 10)                510       \n",
            "=================================================================\n",
            "Total params: 84,060\n",
            "Trainable params: 84,060\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "syqStgrikjzG",
        "outputId": "dadc5196-d2cb-48dc-aa59-9e37892c726b"
      },
      "source": [
        "model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'])\n",
        "model.fit(X_train,y_train,epochs=10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/10\n",
            "1875/1875 [==============================] - 4s 2ms/step - loss: 0.3331 - accuracy: 0.9022\n",
            "Epoch 2/10\n",
            "1875/1875 [==============================] - 4s 2ms/step - loss: 0.1841 - accuracy: 0.9472\n",
            "Epoch 3/10\n",
            "1875/1875 [==============================] - 4s 2ms/step - loss: 0.1289 - accuracy: 0.9637\n",
            "Epoch 4/10\n",
            "1875/1875 [==============================] - 4s 2ms/step - loss: 0.1040 - accuracy: 0.9697\n",
            "Epoch 5/10\n",
            "1875/1875 [==============================] - 4s 2ms/step - loss: 0.0785 - accuracy: 0.9779\n",
            "Epoch 6/10\n",
            "1875/1875 [==============================] - 4s 2ms/step - loss: 0.0634 - accuracy: 0.9829\n",
            "Epoch 7/10\n",
            "1875/1875 [==============================] - 4s 2ms/step - loss: 0.0499 - accuracy: 0.9860\n",
            "Epoch 8/10\n",
            "1875/1875 [==============================] - 4s 2ms/step - loss: 0.0395 - accuracy: 0.9889\n",
            "Epoch 9/10\n",
            "1875/1875 [==============================] - 4s 2ms/step - loss: 0.0353 - accuracy: 0.9906\n",
            "Epoch 10/10\n",
            "1875/1875 [==============================] - 4s 2ms/step - loss: 0.0266 - accuracy: 0.9931\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7fe640f79a90>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EbBkJyTQnKBG"
      },
      "source": [
        "## Comparing accuracy of fine-tuned and non fine-tuned classifier"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xCdpuiV-lUDU",
        "outputId": "585e9c8d-641c-4213-d087-975f0256a2c6"
      },
      "source": [
        "print(\"Accuracy without fine-tuning: {:.2f}\".format(clf.evaluate(X_test_clf,y_test)[1]))\n",
        "print(\"Accuracy with fine-tuning: {:.2f}\".format(model.evaluate(X_test,y_test)[1]))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "313/313 [==============================] - 1s 2ms/step - loss: 0.6333 - accuracy: 0.7933\n",
            "Accuracy without fine-tuning: 0.79\n",
            "313/313 [==============================] - 1s 2ms/step - loss: 0.0910 - accuracy: 0.9729\n",
            "Accuracy with fine-tuning: 0.97\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M1JdudiHltjy"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}