Gravity Optimizer Benchmarks (Tensorflow).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "gravity_optimizer_notebook.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ROo3YHH_377X"
      },
      "source": [
        "# Import Modules"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fgVgFp7kCmdf"
      },
      "source": [
        "!git clone https://github.com/dariush-bahrami/gravity.optimizer.git"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0siAij-6WGyE"
      },
      "source": [
        "import sys\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from matplotlib import pyplot as plt\n",
        "plt.style.use('dark_background')\n",
        "\n",
        "device_name = tf.test.gpu_device_name()\n",
        "if device_name != '/device:GPU:0':\n",
        "    Print('GPU device not found')\n",
        "else:\n",
        "    print('Found GPU at: {}'.format(device_name))\n",
        "\n",
        "sys.path.append('/content/gravity.optimizer')\n",
        "import gravity"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pRm4P1XNXzA_"
      },
      "source": [
        "# Optimizer Definition"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kbP6IdVzieJj"
      },
      "source": [
        "class Gravity(tf.keras.optimizers.Optimizer):\n",
        "    def __init__(self,\n",
        "                 learning_rate=0.1,\n",
        "                 alpha=0.01,\n",
        "                 beta=0.9,\n",
        "                 name=\"Gravity\",\n",
        "                 **kwargs):\n",
        "        super(Gravity, self).__init__(name, **kwargs)\n",
        "        self._set_hyper('learning_rate', kwargs.get('lr', learning_rate))\n",
        "        self._set_hyper('decay', self._initial_decay)\n",
        "        self._set_hyper('alpha', alpha)\n",
        "        self._set_hyper('beta', beta)\n",
        "        self.epsilon = 1e-7\n",
        "\n",
        "    def _create_slots(self, var_list):\n",
        "        alpha = self._get_hyper(\"alpha\")\n",
        "        stddev = alpha / self.learning_rate\n",
        "        initializer = tf.keras.initializers.RandomNormal(mean=0.0,\n",
        "                                                         stddev=stddev,\n",
        "                                                         seed=None)\n",
        "        for var in var_list:\n",
        "            self.add_slot(var, \"velocity\", initializer=initializer)\n",
        "\n",
        "    @tf.function\n",
        "    def _resource_apply_dense(self, grad, var):\n",
        "        # Get Data\n",
        "        var_dtype = var.dtype.base_dtype\n",
        "        lr_t = self._decayed_lr(var_dtype)\n",
        "        beta = self._get_hyper(\"beta\", var_dtype)\n",
        "        t = tf.cast(self.iterations, float)\n",
        "        beta_hat = (beta * t + 1) / (t + 2)\n",
        "        velocity = self.get_slot(var, \"velocity\")\n",
        "\n",
        "        # Calculations\n",
        "        max_step_grad = 1 / tf.math.reduce_max(tf.math.abs(grad))\n",
        "        gradient_term = grad / (1 + (grad / max_step_grad)**2)\n",
        "\n",
        "        # update variables\n",
        "        updated_velocity = velocity.assign(beta_hat * velocity +\n",
        "                                           (1 - beta_hat) * gradient_term)\n",
        "        updated_var = var.assign(var - lr_t * updated_velocity)\n",
        "\n",
        "        # updates = [updated_var, updated_velocity]\n",
        "        # return tf.group(*updates)\n",
        "    def _resource_apply_sparse(self, grad, var):\n",
        "        raise NotImplementedError\n",
        "\n",
        "    def get_config(self):\n",
        "        config = super(Gravity, self).get_config()\n",
        "        config.update({\n",
        "            'learning_rate':\n",
        "            self._serialize_hyperparameter('learning_rate'),\n",
        "            'decay':\n",
        "            self._serialize_hyperparameter('decay'),\n",
        "            'alpha':\n",
        "            self._serialize_hyperparameter('alpha'),\n",
        "            'beta':\n",
        "            self._serialize_hyperparameter('beta'),\n",
        "            'epsilon':\n",
        "            self.epsilon,\n",
        "        })\n",
        "        return config"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mj1fRGlb6V3K"
      },
      "source": [
        "# Benchmarks"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1fzsIQBedqPG"
      },
      "source": [
        "## MNIST"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XurAPLKoyz87"
      },
      "source": [
        "mnist_dict = gravity.get_dataset_mnist(verbose=True, show_images=True)\n",
        "x_train_mnist, y_train_mnist = mnist_dict['train_data']\n",
        "x_test_mnist, y_test_mnist = mnist_dict['test_data']\n",
        "mnist_classes = mnist_dict['classes']\n",
        "mnist_input_shape = mnist_dict['input_shape']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g5RN-k1AycU5"
      },
      "source": [
        "### VGG16"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L3Cf85GPdqPH"
      },
      "source": [
        "batch_size = 128\n",
        "epochs = 10\n",
        "\n",
        "gravity_opt = Gravity(learning_rate=0.1, alpha=0.01, beta=0.9)\n",
        "adam_opt = tf.keras.optimizers.Adam(learning_rate=0.00025)\n",
        "rmsprop_opt = tf.keras.optimizers.RMSprop(learning_rate=0.0001)\n",
        "\n",
        "cost_func = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "\n",
        "base_model = gravity.get_vgg16(classes=mnist_classes,\n",
        "                               input_shape=(32, 32, 1),\n",
        "                               drop_out_rate=0,\n",
        "                               print_summary=False)\n",
        "model = gravity.resize_model_input_size(base_model,\n",
        "                                        target_size=(32, 32),\n",
        "                                        data_shape=(28, 28, 1),\n",
        "                                        print_summary=False)\n",
        "\n",
        "# Try different optimizers by replacing gravity_opt\n",
        "optimizer = gravity_opt\n",
        "model.compile(optimizer=optimizer,\n",
        "              loss=cost_func,\n",
        "              metrics=['accuracy'])\n",
        "history = model.fit(x_train_mnist, y_train_mnist,\n",
        "                    batch_size=batch_size,\n",
        "                    epochs=epochs,\n",
        "                    validation_data=(x_test_mnist, y_test_mnist))\n",
        "gravity.plot_history(history)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1xUt9xvndqPI"
      },
      "source": [
        "comment = ''\n",
        "gravity.save_run_info(model,\n",
        "                      history,\n",
        "                      optimizer,\n",
        "                      'MNIST',\n",
        "                      comment=comment)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lGuLy-eFlK2R"
      },
      "source": [
        "### VGG19"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_utgLHPJlK2R"
      },
      "source": [
        "batch_size = 128\n",
        "epochs = 10\n",
        "\n",
        "gravity_opt = Gravity(learning_rate=0.1, alpha=0.01, beta=0.9)\n",
        "adam_opt = tf.keras.optimizers.Adam(learning_rate=0.00025)\n",
        "rmsprop_opt = tf.keras.optimizers.RMSprop(learning_rate=0.001)\n",
        "\n",
        "cost_func = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "\n",
        "base_model = gravity.get_vgg19(classes=mnist_classes,\n",
        "                               input_shape=(32, 32, 1),\n",
        "                               drop_out_rate=0,\n",
        "                               print_summary=False)\n",
        "model = gravity.resize_model_input_size(base_model,\n",
        "                                        target_size=(32, 32),\n",
        "                                        data_shape=(28, 28, 1),\n",
        "                                        print_summary=False)\n",
        "optimizer = gravity_opt\n",
        "model.compile(optimizer=optimizer,\n",
        "              loss=cost_func,\n",
        "              metrics=['accuracy'])\n",
        "history = model.fit(x_train_mnist, y_train_mnist,\n",
        "                    batch_size=batch_size,\n",
        "                    epochs=epochs,\n",
        "                    validation_data=(x_test_mnist, y_test_mnist))\n",
        "gravity.plot_history(history)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Et8UxnE_lK2S"
      },
      "source": [
        "comment = ''\n",
        "gravity.save_run_info(model,\n",
        "                      history,\n",
        "                      optimizer,\n",
        "                      'MNIST',\n",
        "                      comment=comment)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ma2-qYW1sdpn"
      },
      "source": [
        "## Fashion MNIST"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pSCVLXVzsdpo"
      },
      "source": [
        "fashion_mnist_dict = gravity.get_dataset_fashion_mnist(verbose=True, show_images=True)\n",
        "x_train_fashion_mnist, y_train_fashion_mnist = fashion_mnist_dict['train_data']\n",
        "x_test_fashion_mnist, y_test_fashion_mnist = fashion_mnist_dict['test_data']\n",
        "fashion_mnist_classes = fashion_mnist_dict['classes']\n",
        "fashion_mnist_input_shape = fashion_mnist_dict['input_shape']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WUXoqDldsdpp"
      },
      "source": [
        "### VGG16"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1OrCBYxhsdpp"
      },
      "source": [
        "batch_size = 128\n",
        "epochs = 10\n",
        "\n",
        "gravity_opt = Gravity(learning_rate=0.1, alpha=0.01, beta=0.9)\n",
        "adam_opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n",
        "rmsprop_opt = tf.keras.optimizers.RMSprop(learning_rate=0.001)\n",
        "\n",
        "cost_func = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "\n",
        "base_model = gravity.get_vgg16(classes=fashion_mnist_classes,\n",
        "                               input_shape=(32, 32, 1),\n",
        "                               drop_out_rate=0,\n",
        "                               print_summary=False)\n",
        "model = gravity.resize_model_input_size(base_model,\n",
        "                                        target_size=(32, 32),\n",
        "                                        data_shape=fashion_mnist_input_shape,\n",
        "                                        print_summary=False)\n",
        "\n",
        "optimizer = gravity_opt\n",
        "model.compile(optimizer=optimizer,\n",
        "            loss=cost_func,\n",
        "            metrics=['accuracy'])\n",
        "history = model.fit(x_train_fashion_mnist, y_train_fashion_mnist,\n",
        "                    batch_size=batch_size,\n",
        "                    epochs=epochs,\n",
        "                    validation_data=(x_test_fashion_mnist, y_test_fashion_mnist))\n",
        "gravity.plot_history(history)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J9r6WTBXsdpp"
      },
      "source": [
        "comment = ''\n",
        "gravity.save_run_info(model,\n",
        "                      history,\n",
        "                      optimizer,\n",
        "                      'Fashion MNIST',\n",
        "                      comment=comment)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9ByXsh_Asdpq"
      },
      "source": [
        "### VGG19"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qdgPW3Zcsdpq"
      },
      "source": [
        "batch_size = 128\n",
        "epochs = 10\n",
        "\n",
        "gravity_opt = Gravity(learning_rate=0.1, alpha=0.01, beta=0.9)\n",
        "adam_opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n",
        "rmsprop_opt = tf.keras.optimizers.RMSprop(learning_rate=0.001)\n",
        "\n",
        "cost_func = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "\n",
        "base_model = gravity.get_vgg19(classes=fashion_mnist_classes,\n",
        "                               input_shape=(32, 32, 1),\n",
        "                               drop_out_rate=0,\n",
        "                               print_summary=False)\n",
        "model = gravity.resize_model_input_size(base_model,\n",
        "                                        target_size=(32, 32),\n",
        "                                        data_shape=fashion_mnist_input_shape,\n",
        "                                        print_summary=False)\n",
        "\n",
        "optimizer = gravity_opt\n",
        "model.compile(optimizer=optimizer,\n",
        "            loss=cost_func,\n",
        "            metrics=['accuracy'])\n",
        "history = model.fit(x_train_fashion_mnist, y_train_fashion_mnist,\n",
        "                    batch_size=batch_size,\n",
        "                    epochs=epochs,\n",
        "                    validation_data=(x_test_fashion_mnist, y_test_fashion_mnist))\n",
        "gravity.plot_history(history)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6c_wHIWwsdpq"
      },
      "source": [
        "comment = ''\n",
        "gravity.save_run_info(model,\n",
        "                      history,\n",
        "                      optimizer,\n",
        "                      'Fashion MNIST',\n",
        "                      comment=comment)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "empswep92qZf"
      },
      "source": [
        "## CIFAR10"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m-gbUv7Y2qZf"
      },
      "source": [
        "cifar10_dict = gravity.get_dataset_cifar10(verbose=True, show_images=True)\n",
        "x_train_cifar10, y_train_cifar10 = cifar10_dict['train_data']\n",
        "x_test_cifar10, y_test_cifar10 = cifar10_dict['test_data']\n",
        "cifar10_classes = cifar10_dict['classes']\n",
        "cifar10_input_shape = cifar10_dict['input_shape']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "URz5gjH72qZg"
      },
      "source": [
        "### VGG16"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tFseoFTb2qZg"
      },
      "source": [
        "batch_size = 128\n",
        "epochs = 10\n",
        "\n",
        "gravity_opt = Gravity(learning_rate=0.1, alpha=0.01, beta=0.9)\n",
        "adam_opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n",
        "rmsprop_opt = tf.keras.optimizers.RMSprop(learning_rate=0.001)\n",
        "\n",
        "cost_func = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "\n",
        "model = gravity.get_vgg16(classes=cifar10_classes,\n",
        "                          input_shape=(32, 32, 3),\n",
        "                          drop_out_rate=0,\n",
        "                          print_summary=False)\n",
        "\n",
        "optimizer = gravity_opt\n",
        "model.compile(optimizer=optimizer,\n",
        "            loss=cost_func,\n",
        "            metrics=['accuracy'])\n",
        "history = model.fit(x_train_cifar10, y_train_cifar10,\n",
        "                    batch_size=batch_size,\n",
        "                    epochs=epochs,\n",
        "                    validation_data=(x_test_cifar10, y_test_cifar10))\n",
        "gravity.plot_history(history)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dwP5qMza2qZg"
      },
      "source": [
        "comment = ''\n",
        "gravity.save_run_info(model,\n",
        "                      history,\n",
        "                      optimizer,\n",
        "                      'CIFAR10',\n",
        "                      comment=comment)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ooM3AFJq8PlF"
      },
      "source": [
        "### VGG19\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_9GM1MEL8PlG"
      },
      "source": [
        "batch_size = 128\n",
        "epochs = 10\n",
        "\n",
        "gravity_opt = Gravity(learning_rate=0.1, alpha=0.01, beta=0.9)\n",
        "adam_opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n",
        "rmsprop_opt = tf.keras.optimizers.RMSprop(learning_rate=0.001)\n",
        "\n",
        "cost_func = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "\n",
        "model = gravity.get_vgg19(classes=cifar10_classes,\n",
        "                          input_shape=(32, 32, 3),\n",
        "                          drop_out_rate=0,\n",
        "                          print_summary=False)\n",
        "\n",
        "optimizer = gravity_opt\n",
        "model.compile(optimizer=optimizer,\n",
        "            loss=cost_func,\n",
        "            metrics=['accuracy'])\n",
        "\n",
        "history = model.fit(x_train_cifar10, y_train_cifar10,\n",
        "                    batch_size=batch_size,\n",
        "                    epochs=epochs,\n",
        "                    validation_data=(x_test_cifar10, y_test_cifar10))\n",
        "gravity.plot_history(history)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yfuz9QUy8PlG"
      },
      "source": [
        "comment = ''\n",
        "gravity.save_run_info(model,\n",
        "                      history,\n",
        "                      optimizer,\n",
        "                      'CIFAR10',\n",
        "                      comment=comment)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "AUCxa8zG0JTU"
      },
      "source": [
        "## CIFAR100 (Coarse)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wb_rXfpa0JTV"
      },
      "source": [
        "cifar100_dict = gravity.get_dataset_cifar100(label_mode='coarse', verbose=True, show_images=True)\n",
        "x_train_cifar100, y_train_cifar100 = cifar100_dict['train_data']\n",
        "x_test_cifar100, y_test_cifar100 = cifar100_dict['test_data']\n",
        "cifar100_classes = cifar100_dict['classes']\n",
        "cifar100_input_shape = cifar100_dict['input_shape']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "r0YX6t4g0JTV"
      },
      "source": [
        "### VGG16"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DEFSQP0B0JTV"
      },
      "source": [
        "batch_size = 128\n",
        "epochs = 10\n",
        "\n",
        "gravity_opt = Gravity(learning_rate=0.1, alpha=0.01, beta=0.9)\n",
        "adam_opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n",
        "rmsprop_opt = tf.keras.optimizers.RMSprop(learning_rate=0.001)\n",
        "\n",
        "cost_func = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "\n",
        "model = gravity.get_vgg16(classes=cifar100_classes,\n",
        "                          input_shape=(32, 32, 3),\n",
        "                          drop_out_rate=0,\n",
        "                          print_summary=False)\n",
        "\n",
        "optimizer = gravity_opt\n",
        "model.compile(optimizer=optimizer,\n",
        "            loss=cost_func,\n",
        "            metrics=['accuracy'])\n",
        "\n",
        "history = model.fit(x_train_cifar100, y_train_cifar100,\n",
        "                    batch_size=batch_size,\n",
        "                    epochs=epochs,\n",
        "                    validation_data=(x_test_cifar100, y_test_cifar100))\n",
        "gravity.plot_history(history)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YFPgUWT80JTV"
      },
      "source": [
        "comment = ''\n",
        "gravity.save_run_info(model,\n",
        "                      history,\n",
        "                      optimizer,\n",
        "                      'CIFAR100 (Coarse)',\n",
        "                      comment=comment)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JbqmgPLT0JTX"
      },
      "source": [
        "### VGG19\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MjrVpTPI0JTX"
      },
      "source": [
        "batch_size = 128\n",
        "epochs = 10\n",
        "\n",
        "gravity_opt = Gravity(learning_rate=0.1, alpha=0.01, beta=0.9)\n",
        "adam_opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n",
        "rmsprop_opt = tf.keras.optimizers.RMSprop(learning_rate=0.001)\n",
        "\n",
        "cost_func = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "\n",
        "model = gravity.get_vgg19(classes=cifar100_classes,\n",
        "                          input_shape=(32, 32, 3),\n",
        "                          drop_out_rate=0,\n",
        "                          print_summary=False)\n",
        "\n",
        "optimizer = gravity_opt\n",
        "model.compile(optimizer=optimizer,\n",
        "            loss=cost_func,\n",
        "            metrics=['accuracy'])\n",
        "\n",
        "history = model.fit(x_train_cifar100, y_train_cifar100,\n",
        "                    batch_size=batch_size,\n",
        "                    epochs=epochs,\n",
        "                    validation_data=(x_test_cifar100, y_test_cifar100))\n",
        "gravity.plot_history(history)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kqUpUjnI0JTX"
      },
      "source": [
        "comment = ''\n",
        "gravity.save_run_info(model,\n",
        "                      history,\n",
        "                      optimizer,\n",
        "                      'CIFAR100 (Coarse)',\n",
        "                      comment=comment)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jE3CtpQT1XzB"
      },
      "source": [
        "## CIFAR100 (Fine)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4TCya3tN1XzB"
      },
      "source": [
        "cifar100_dict = gravity.get_dataset_cifar100(label_mode='fine', verbose=True, show_images=True)\n",
        "x_train_cifar100, y_train_cifar100 = cifar100_dict['train_data']\n",
        "x_test_cifar100, y_test_cifar100 = cifar100_dict['test_data']\n",
        "cifar100_classes = cifar100_dict['classes']\n",
        "cifar100_input_shape = cifar100_dict['input_shape']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cbPYUYlK1XzC"
      },
      "source": [
        "### VGG16"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2wfn3zZb1XzC"
      },
      "source": [
        "batch_size = 128\n",
        "epochs = 10\n",
        "\n",
        "gravity_opt = Gravity(learning_rate=0.1, alpha=0.01, beta=0.9)\n",
        "adam_opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n",
        "rmsprop_opt = tf.keras.optimizers.RMSprop(learning_rate=0.001)\n",
        "\n",
        "cost_func = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "\n",
        "model = gravity.get_vgg16(classes=cifar100_classes,\n",
        "                          input_shape=(32, 32, 3),\n",
        "                          drop_out_rate=0,\n",
        "                          print_summary=False)\n",
        "\n",
        "optimizer = gravity_opt\n",
        "model.compile(optimizer=optimizer,\n",
        "            loss=cost_func,\n",
        "            metrics=['accuracy'])\n",
        "history = model.fit(x_train_cifar100, y_train_cifar100,\n",
        "                    batch_size=batch_size,\n",
        "                    epochs=epochs,\n",
        "                    validation_data=(x_test_cifar100, y_test_cifar100))\n",
        "gravity.plot_history(history)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "92t1RFNw1XzC"
      },
      "source": [
        "comment = ''\n",
        "gravity.save_run_info(model,\n",
        "                      history,\n",
        "                      optimizer,\n",
        "                      'CIFAR100 (Fine)',\n",
        "                      comment=comment)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "p9_f-gMU1XzD"
      },
      "source": [
        "### VGG19\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aIuFd8Cw1XzE"
      },
      "source": [
        "batch_size = 128\n",
        "epochs = 10\n",
        "\n",
        "gravity_opt = Gravity(learning_rate=0.1, alpha=0.01, beta=0.9)\n",
        "adam_opt = tf.keras.optimizers.Adam(learning_rate=0.001)\n",
        "rmsprop_opt = tf.keras.optimizers.RMSprop(learning_rate=0.001)\n",
        "\n",
        "cost_func = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)\n",
        "\n",
        "model = gravity.get_vgg19(classes=cifar100_classes,\n",
        "                          input_shape=(32, 32, 3),\n",
        "                          drop_out_rate=0,\n",
        "                          print_summary=False)\n",
        "\n",
        "optimizer = gravity_opt\n",
        "model.compile(optimizer=optimizer,\n",
        "            loss=cost_func,\n",
        "            metrics=['accuracy'])\n",
        "history = model.fit(x_train_cifar100, y_train_cifar100,\n",
        "                    batch_size=batch_size,\n",
        "                    epochs=epochs,\n",
        "                    validation_data=(x_test_cifar100, y_test_cifar100))\n",
        "gravity.plot_history(history)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h0OWSTit1XzE"
      },
      "source": [
        "comment = ''\n",
        "gravity.save_run_info(model,\n",
        "                     history,\n",
        "                     optimizer,\n",
        "                     'CIFAR100 (Fine)',\n",
        "                     comment=comment)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}