NN.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "q11m2EVupsUO"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "d-F8Sm_rtAtH"
      },
      "outputs": [],
      "source": [
        "data = keras.datasets.fashion_mnist"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "qkf8kk8Kt1pu"
      },
      "outputs": [],
      "source": [
        "(train_images, train_labels), (test_images, test_labels) = data.load_data()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "yKjuEMeJt-Pm"
      },
      "outputs": [],
      "source": [
        "class_names = ['t_shirt', 'trouser', 'pullover', 'dress', 'coat',\n",
        "          'sandal', 'shirt', 'sneaker', 'bag', 'ankle_boots']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "cKGBhJhpuxiQ"
      },
      "outputs": [],
      "source": [
        "from tensorflow.python.ops import control_flow_ops\n",
        "from tensorflow.python.ops import math_ops\n",
        "from tensorflow.python.ops import state_ops\n",
        "from tensorflow.python.framework import ops\n",
        "from tensorflow.python.training import optimizer\n",
        "#from tensorflow.python.eager import context\n",
        "from tensorflow.python.ops import resource_variable_ops\n",
        "from tensorflow.python.ops import variable_scope\n",
        "from tensorflow.python.training import training_ops\n",
        "\n",
        "from keras import backend_config\n",
        "from keras.optimizers.optimizer_v2 import optimizer_v2\n",
        "\n",
        "class AdamOptimizer(optimizer_v2.OptimizerV2):\n",
        "    def __init__(\n",
        "        self,\n",
        "        learning_rate=0.001,\n",
        "        beta_1=0.9,\n",
        "        beta_2=0.999,\n",
        "        epsilon=1e-7,\n",
        "        name=\"Adam\",\n",
        "        **kwargs\n",
        "    ):\n",
        "        super().__init__(name, **kwargs)\n",
        "        self._set_hyper(\"learning_rate\", kwargs.get(\"lr\", learning_rate))\n",
        "        self._set_hyper(\"decay\", self._initial_decay)\n",
        "        self._set_hyper(\"beta_1\", beta_1)\n",
        "        self._set_hyper(\"beta_2\", beta_2)\n",
        "        self.epsilon = epsilon or backend_config.epsilon()\n",
        "\n",
        "    def _create_slots(self, var_list):\n",
        "        for var in var_list:\n",
        "            self.add_slot(var, \"m\")\n",
        "        for var in var_list:\n",
        "            self.add_slot(var, \"v\")\n",
        "\n",
        "    def _prepare_local(self, var_device, var_dtype, apply_state):\n",
        "        super()._prepare_local(var_device, var_dtype, apply_state)\n",
        "\n",
        "        local_step = tf.cast(self.iterations + 1, var_dtype)\n",
        "        beta_1_t = tf.identity(self._get_hyper(\"beta_1\", var_dtype))\n",
        "        beta_2_t = tf.identity(self._get_hyper(\"beta_2\", var_dtype))\n",
        "        beta_1_power = tf.pow(beta_1_t, local_step)\n",
        "        beta_2_power = tf.pow(beta_2_t, local_step)\n",
        "        stepSizeUpperBoundParameter = tf.sqrt(1 - beta_2_power) / (1 - beta_1_power)\n",
        "        lr = apply_state[(var_device, var_dtype)][\"lr_t\"] * (\n",
        "            stepSizeUpperBoundParameter\n",
        "        )\n",
        "        apply_state[(var_device, var_dtype)].update(\n",
        "            dict(\n",
        "                lr=lr,\n",
        "                epsilon=tf.convert_to_tensor(self.epsilon, var_dtype),\n",
        "                beta_1_t=beta_1_t,\n",
        "                beta_1_power=beta_1_power,\n",
        "                one_minus_beta_1_t=1 - beta_1_t,\n",
        "                beta_2_t=beta_2_t,\n",
        "                beta_2_power=beta_2_power,\n",
        "                one_minus_beta_2_t=1 - beta_2_t,\n",
        "            )\n",
        "        )\n",
        "\n",
        "    def set_weights(self, weights):\n",
        "        params = self.weights\n",
        "        num_vars = int((len(params) - 1) / 2)\n",
        "        total_vars = 3 * num_vars + 1\n",
        "        newWeights = []\n",
        "        if len(weights) == total_vars:\n",
        "            for i in range(0,len(params)):\n",
        "              newWeights.append(weights[i])\n",
        "        super().set_weights(newWeights)\n",
        "\n",
        "    def _resource_apply_dense(self, grad, var, apply_state=None):\n",
        "        var_device, var_dtype = var.device, var.dtype.base_dtype\n",
        "        coefficients = (apply_state or {}).get(\n",
        "            (var_device, var_dtype)\n",
        "        ) or self._fallback_apply_state(var_device, var_dtype)\n",
        "\n",
        "        m = self.get_slot(var, \"m\")\n",
        "        v = self.get_slot(var, \"v\")\n",
        "\n",
        "\n",
        "        return tf.raw_ops.ResourceApplyAdam(\n",
        "            var=var.handle,\n",
        "            m=m.handle,\n",
        "            v=v.handle,\n",
        "            beta1_power=coefficients[\"beta_1_power\"],\n",
        "            beta2_power=coefficients[\"beta_2_power\"],\n",
        "            lr=coefficients[\"lr_t\"],\n",
        "            beta1=coefficients[\"beta_1_t\"],\n",
        "            beta2=coefficients[\"beta_2_t\"],\n",
        "            epsilon=coefficients[\"epsilon\"],\n",
        "            grad=grad,\n",
        "            use_locking=self._use_locking,\n",
        "        )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "id": "-7ibdT68uyce"
      },
      "outputs": [],
      "source": [
        "def optimizer_fn(optimizer, lr, name='Optimizer'):\n",
        "    with tf.compat.v1.variable_scope(name):\n",
        "        global_step = tf.Variable(1, dtype=tf.float32, trainable=False)\n",
        "        cur_lr = lr / tf.math.sqrt(x=global_step)\n",
        "\n",
        "        if optimizer == 'SGDNesterov':\n",
        "            return tf.keras.optimizers.SGD(learning_rate=lr,momentum=0.99,nesterov=True)\n",
        "        elif optimizer == 'Adagrad':\n",
        "            return tf.keras.optimizers.Adagrad(learning_rate=cur_lr)\n",
        "        elif optimizer == 'RMSProp':\n",
        "            return tf.keras.optimizers.RMSprop(learning_rate=cur_lr)\n",
        "        elif optimizer == 'AdaDelta':\n",
        "            return tf.keras.optimizers.Adadelta(learning_rate=cur_lr)\n",
        "        elif optimizer == 'Adam':\n",
        "            return AdamOptimizer(learning_rate=cur_lr)\n",
        "        else:\n",
        "            raise NotImplementedError(\" [*] Optimizer is not included in list!\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "LrzEAi7uu0ou"
      },
      "outputs": [],
      "source": [
        "train_images = train_images/255.0\n",
        "test_images = test_images/255.0"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "id": "gphSiTY2v_rm"
      },
      "outputs": [],
      "source": [
        "def getModel(dropout,rate=0.2):\n",
        "    if dropout:\n",
        "        model = keras.Sequential([\n",
        "\t\tkeras.layers.Flatten(input_shape=(28,28)),\n",
        "\t\tkeras.layers.Dropout(rate),\n",
        "\t\tkeras.layers.Dense(1000, activation=\"relu\"),\n",
        "\t\tkeras.layers.Dropout(rate),\n",
        "\t\tkeras.layers.Dense(1000, activation=\"relu\"),\n",
        "\t\tkeras.layers.Dropout(rate),\n",
        "\t\tkeras.layers.Dense(10, activation=\"softmax\")\n",
        "\t\t])\n",
        "        return model\n",
        "    else:\n",
        "        model = keras.Sequential([\n",
        "\t\tkeras.layers.Flatten(input_shape=(28,28)),\n",
        "\t\tkeras.layers.Dense(1000, activation=\"relu\"),\n",
        "\t\tkeras.layers.Dense(1000, activation=\"relu\"),\n",
        "\t\tkeras.layers.Dense(10, activation=\"softmax\")\n",
        "\t\t])\n",
        "        return model\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "id": "LYUS-TpUv_rn"
      },
      "outputs": [],
      "source": [
        "def NN(lr):\n",
        "\toptimizer_list=['SGDNesterov','Adagrad','RMSProp','AdaDelta','Adam']\n",
        "\tdropout_list=[True,False]\n",
        "\tfor optimizer in optimizer_list:\n",
        "\t\tfor dropout in dropout_list:\n",
        "\t\t\tprint(\"Optimizer: \",optimizer)\n",
        "\t\t\tprint(\"Dropout: \",dropout)\n",
        "\t\t\tmodel = getModel(dropout)\n",
        "\t\t\tmodel.compile(optimizer=optimizer_fn(optimizer, lr), loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
        "\t\t\tmodel.fit(train_images, train_labels, epochs=20)\n",
        "\t\t\ttest_loss, test_acc = model.evaluate(test_images, test_labels)\n",
        "\t\t\tprint('Test accuracy:', test_acc)\n",
        "\t\t\tprint(\"===========================================\")\n",
        "\n",
        "\t"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0guerSOOv_rn",
        "outputId": "a4e2304d-d021-4f3b-b75d-adb86c9383ed"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Optimizer:  SGDNesterov\n",
            "Dropout:  True\n",
            "1875/1875 [==============================] - 39s 20ms/step - loss: 0.5979 - accuracy: 0.7848\n",
            "313/313 [==============================] - 2s 6ms/step - loss: 0.4630 - accuracy: 0.8283\n",
            "Test accuracy: 0.8282999992370605\n",
            "===========================================\n",
            "Optimizer:  SGDNesterov\n",
            "Dropout:  False\n",
            "1875/1875 [==============================] - 37s 19ms/step - loss: 0.5150 - accuracy: 0.8181\n",
            "313/313 [==============================] - 2s 8ms/step - loss: 0.4094 - accuracy: 0.8504\n",
            "Test accuracy: 0.8503999710083008\n",
            "===========================================\n",
            "Optimizer:  Adagrad\n",
            "Dropout:  True\n",
            "1875/1875 [==============================] - 39s 20ms/step - loss: 1.0557 - accuracy: 0.6544\n",
            "313/313 [==============================] - 2s 7ms/step - loss: 0.6963 - accuracy: 0.7516\n",
            "Test accuracy: 0.7516000270843506\n",
            "===========================================\n",
            "Optimizer:  Adagrad\n",
            "Dropout:  False\n",
            "1875/1875 [==============================] - 36s 19ms/step - loss: 0.9105 - accuracy: 0.7266\n",
            "313/313 [==============================] - 2s 7ms/step - loss: 0.6599 - accuracy: 0.7816\n",
            "Test accuracy: 0.7815999984741211\n",
            "===========================================\n",
            "Optimizer:  RMSProp\n",
            "Dropout:  True\n",
            "1875/1875 [==============================] - 51s 27ms/step - loss: 0.5953 - accuracy: 0.7899\n",
            "313/313 [==============================] - 2s 7ms/step - loss: 0.5231 - accuracy: 0.8178\n",
            "Test accuracy: 0.817799985408783\n",
            "===========================================\n",
            "Optimizer:  RMSProp\n",
            "Dropout:  False\n",
            "1875/1875 [==============================] - 47s 25ms/step - loss: 0.5291 - accuracy: 0.8131\n",
            "313/313 [==============================] - 2s 7ms/step - loss: 0.4799 - accuracy: 0.8409\n",
            "Test accuracy: 0.8409000039100647\n",
            "===========================================\n",
            "Optimizer:  AdaDelta\n",
            "Dropout:  True\n",
            "1875/1875 [==============================] - 40s 21ms/step - loss: 2.0144 - accuracy: 0.3500\n",
            "313/313 [==============================] - 2s 6ms/step - loss: 1.6144 - accuracy: 0.6089\n",
            "Test accuracy: 0.6089000105857849\n",
            "===========================================\n",
            "Optimizer:  AdaDelta\n",
            "Dropout:  False\n",
            "1875/1875 [==============================] - 43s 23ms/step - loss: 1.7680 - accuracy: 0.5195\n",
            "313/313 [==============================] - 2s 7ms/step - loss: 1.3724 - accuracy: 0.6382\n",
            "Test accuracy: 0.6381999850273132\n",
            "===========================================\n",
            "Optimizer:  Adam\n",
            "Dropout:  True\n",
            "1875/1875 [==============================] - 40s 21ms/step - loss: 0.5366 - accuracy: 0.8026\n",
            "313/313 [==============================] - 2s 7ms/step - loss: 0.4269 - accuracy: 0.8465\n",
            "Test accuracy: 0.8464999794960022\n",
            "===========================================\n",
            "Optimizer:  Adam\n",
            "Dropout:  False\n",
            "1875/1875 [==============================] - 41s 21ms/step - loss: 0.4651 - accuracy: 0.8315\n",
            "313/313 [==============================] - 2s 7ms/step - loss: 0.4109 - accuracy: 0.8561\n",
            "Test accuracy: 0.8561000227928162\n",
            "===========================================\n"
          ]
        }
      ],
      "source": [
        "NN(0.001)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gm9yBVjxv_rn",
        "outputId": "622a489d-6906-4119-b323-f1ac34555cdb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Optimizer:  SGDNesterov\n",
            "Dropout:  True\n",
            "1875/1875 [==============================] - 39s 21ms/step - loss: 0.8117 - accuracy: 0.7046\n",
            "313/313 [==============================] - 2s 7ms/step - loss: 0.7455 - accuracy: 0.7172\n",
            "Test accuracy: 0.717199981212616\n",
            "===========================================\n",
            "Optimizer:  SGDNesterov\n",
            "Dropout:  False\n",
            "1875/1875 [==============================] - 37s 19ms/step - loss: 0.5372 - accuracy: 0.8112\n",
            "313/313 [==============================] - 2s 7ms/step - loss: 0.5638 - accuracy: 0.8170\n",
            "Test accuracy: 0.8169999718666077\n",
            "===========================================\n",
            "Optimizer:  Adagrad\n",
            "Dropout:  True\n",
            "1875/1875 [==============================] - 37s 20ms/step - loss: 0.6359 - accuracy: 0.7774\n",
            "313/313 [==============================] - 2s 7ms/step - loss: 0.4971 - accuracy: 0.8214\n",
            "Test accuracy: 0.821399986743927\n",
            "===========================================\n",
            "Optimizer:  Adagrad\n",
            "Dropout:  False\n",
            "1875/1875 [==============================] - 39s 21ms/step - loss: 0.5530 - accuracy: 0.8091\n",
            "313/313 [==============================] - 2s 7ms/step - loss: 0.4732 - accuracy: 0.8334\n",
            "Test accuracy: 0.8334000110626221\n",
            "===========================================\n",
            "Optimizer:  RMSProp\n",
            "Dropout:  True\n",
            "1875/1875 [==============================] - 52s 27ms/step - loss: 1.3163 - accuracy: 0.7042\n",
            "313/313 [==============================] - 2s 7ms/step - loss: 0.6263 - accuracy: 0.7763\n",
            "Test accuracy: 0.7763000130653381\n",
            "===========================================\n",
            "Optimizer:  RMSProp\n",
            "Dropout:  False\n",
            "1875/1875 [==============================] - 51s 27ms/step - loss: 1.2463 - accuracy: 0.7605\n",
            "313/313 [==============================] - 2s 7ms/step - loss: 0.5736 - accuracy: 0.8218\n",
            "Test accuracy: 0.8217999935150146\n",
            "===========================================\n",
            "Optimizer:  AdaDelta\n",
            "Dropout:  True\n",
            "1875/1875 [==============================] - 42s 22ms/step - loss: 1.0707 - accuracy: 0.6527\n",
            "313/313 [==============================] - 2s 6ms/step - loss: 0.6791 - accuracy: 0.7542\n",
            "Test accuracy: 0.7541999816894531\n",
            "===========================================\n",
            "Optimizer:  AdaDelta\n",
            "Dropout:  False\n",
            "1875/1875 [==============================] - 40s 21ms/step - loss: 0.8573 - accuracy: 0.7444\n",
            "313/313 [==============================] - 2s 7ms/step - loss: 0.6089 - accuracy: 0.7967\n",
            "Test accuracy: 0.7967000007629395\n",
            "===========================================\n",
            "Optimizer:  Adam\n",
            "Dropout:  True\n",
            "1875/1875 [==============================] - 38s 20ms/step - loss: 0.8812 - accuracy: 0.7048\n",
            "313/313 [==============================] - 2s 7ms/step - loss: 0.6559 - accuracy: 0.7829\n",
            "Test accuracy: 0.7828999757766724\n",
            "===========================================\n",
            "Optimizer:  Adam\n",
            "Dropout:  False\n",
            "1875/1875 [==============================] - 37s 19ms/step - loss: 0.6313 - accuracy: 0.7840\n",
            "313/313 [==============================] - 2s 6ms/step - loss: 0.5334 - accuracy: 0.8164\n",
            "Test accuracy: 0.8163999915122986\n",
            "===========================================\n"
          ]
        }
      ],
      "source": [
        "NN(0.01)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KPm_cGqUwU-v"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3.8.10 64-bit",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.10"
    },
    "vscode": {
      "interpreter": {
        "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1"
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}