Labs/Lab_2A.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Lab_2A.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyMKauX7sgMA0Wpe6/yFnUp7",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "5c919aedc1834306947697dd595adcf4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0e4a7ec8a8fb4dc3812bc8b364db46e7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_5d2e65f38baa4b94a759c847ca828101",
              "IPY_MODEL_68c6b3816ff04abab2cefa2a7d8b7718"
            ]
          }
        },
        "0e4a7ec8a8fb4dc3812bc8b364db46e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5d2e65f38baa4b94a759c847ca828101": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_76ea284e541a4bbba1421a6cde343b00",
            "_dom_classes": [],
            "description": "Epoch 8:   9%",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 100,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 9,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_02c9ce06408d43b1b306550d99527783"
          }
        },
        "68c6b3816ff04abab2cefa2a7d8b7718": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a77d1ba42abf43f783f26a944a5062b4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 9/100 [00:03&lt;00:35,  2.54it/s, loss=2.5]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c844af1092564f83aa9846ebad536ee3"
          }
        },
        "76ea284e541a4bbba1421a6cde343b00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "02c9ce06408d43b1b306550d99527783": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a77d1ba42abf43f783f26a944a5062b4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c844af1092564f83aa9846ebad536ee3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dc40c0293a2c463b8d5089f30201ee45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_36ecf0fa129e46e7a24c481dc1a60c99",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6b1fb292a81b40ab87d30c72fd10060b",
              "IPY_MODEL_1eb4bccd2d134b08bb7002d8dbc00387"
            ]
          }
        },
        "36ecf0fa129e46e7a24c481dc1a60c99": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6b1fb292a81b40ab87d30c72fd10060b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_d96c5bc52dd740bcb347fde0ab37ae71",
            "_dom_classes": [],
            "description": "Epoch 99: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "info",
            "max": 100,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 100,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a9f3385b435941f2a7dd70becaf566c4"
          }
        },
        "1eb4bccd2d134b08bb7002d8dbc00387": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_801d55bc6aed4ed4a3f99303d545d63a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 100/100 [00:45&lt;00:00,  2.21it/s, loss=0.764]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0ca740c0f6e940339e8fd7f37ec1706c"
          }
        },
        "d96c5bc52dd740bcb347fde0ab37ae71": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a9f3385b435941f2a7dd70becaf566c4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "801d55bc6aed4ed4a3f99303d545d63a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0ca740c0f6e940339e8fd7f37ec1706c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/tirtharajdash/BITS-G513_MetaLearning/blob/main/Labs/Lab_2A.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tVV_tpgnfAfA"
      },
      "source": [
        "This notebook has been adapted from NYU's Deep Learning Course taught by Yann LeCun & Alfredo Canziani. [Website](https://atcold.github.io/pytorch-Deep-Learning/)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KibVUNLYfN1t"
      },
      "source": [
        "This notebook covers an example of Gradient Descent, Stochastic Gradient Descent and Backpropagation without using PyTorch's Automatic Gradient Computation Capabilities.\n",
        "\n",
        "A basic explanation of backpropagation was given in last week's lab - [Lab_1A](https://github.com/tirtharajdash/BITS-G513_MetaLearning/blob/main/Lab_1A.ipynb). For an intuitive explanation of Backpropagation check the [CS231n notes on Backpropagation](https://cs231n.github.io/optimization-2/)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VGz6iyM-kG3n"
      },
      "source": [
        "# Gradient Descent"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MPx1UNe_j-aD"
      },
      "source": [
        "## Setup\n",
        "\n",
        "We'll work with the classic MNIST Dataset in this notebook"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q3000HmfdQs5"
      },
      "source": [
        "import numpy as np\n",
        "from sklearn.datasets import fetch_openml\n",
        "mnist = fetch_openml('mnist_784', version=1, cache=True)\n",
        "mnist.target = mnist.target.astype(np.int8)"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eAGNt9DddZjX"
      },
      "source": [
        "X, y = mnist[\"data\"], mnist[\"target\"]"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qHnvDcf8dcjy",
        "outputId": "9a2c565d-f9c6-4309-c5f3-2496bb5119fa"
      },
      "source": [
        "print(X.shape, y.shape)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(70000, 784) (70000,)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tnApz5ORdfDw"
      },
      "source": [
        "X = X / 255"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "48IeWgeWdgfi"
      },
      "source": [
        "digits = 10\n",
        "examples = y.shape[0]\n",
        "\n",
        "y = y.reshape(1, examples)\n",
        "\n",
        "Y_new = np.eye(digits)[y.astype('int32')]\n",
        "Y_new = Y_new.T.reshape(digits, examples)"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "abL9LFF_dgys"
      },
      "source": [
        "m = 60000\n",
        "m_test = X.shape[0] - m\n",
        "\n",
        "X_train, X_test = X[:m].T, X[m:].T\n",
        "Y_train, Y_test = Y_new[:,:m], Y_new[:,m:]\n",
        "\n",
        "shuffle_index = np.random.permutation(m)\n",
        "X_train, Y_train = X_train[:, shuffle_index], Y_train[:, shuffle_index]"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 266
        },
        "id": "9vm3HsOFdihB",
        "outputId": "eca30d8f-354f-4d40-892d-ed897df78a50"
      },
      "source": [
        "import matplotlib\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "i = 1234\n",
        "plt.imshow(X_train[:,i].reshape(28,28), cmap = matplotlib.cm.binary)\n",
        "plt.axis(\"off\")\n",
        "plt.show()\n",
        "Y_train[:,i]"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOcAAADnCAYAAADl9EEgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAEjklEQVR4nO3dMWoUWhSA4TePiUIEiQoJwUYQLCIWLsBWt+BiLNJkH1mCW4iFnYXExlpJkVKxsRmb1zxM7uhMJvOPfl/pYa4X4s8FD0wms9nsH6Dn33VfALiYOCFKnBAlTogSJ0RN58z9Vy6s3uSiP/RyQpQ4IUqcECVOiBInRIkTosQJUeKEKHFClDghSpwQJU6IEidEiROixAlR4oQocUKUOCFKnBAlTogSJ0SJE6LECVHihChxQpQ4IUqcECVOiBInRIkTosQJUeKEKHFClDghSpwQJU6IEidEiROipuu+AC17e3uXzu7cuTP87Nu3b4fzu3fvLnSnv5WXE6LECVHihChxQpQ4IUqcECVOiLLn/Mt8+vRpOP/+/fuls/Pz8+Fnv3z5Mpzbc/4eLydEiROixAlR4oQocUKUOCHKKuUPM2/d8eLFi+F8tA65cePG8LOjNQy/z8sJUeKEKHFClDghSpwQJU6IEidE2XP+Yd6/fz+cf/z4ceGzDw8Ph/NHjx4tfDY/83JClDghSpwQJU6IEidEiROixAlR9pwLOD09Hc6fPHlyTTf52dHR0crOfvjw4crO5mdeTogSJ0SJE6LECVHihChxQpQ4IcqecwHr3GPOc3JyMpxPJpPhfDq9/J/E7du3F7oTi/FyQpQ4IUqcECVOiBInRIkTosQJUfacG+b169fD+bw95rz5vXv3Lp09f/58+FmulpcTosQJUeKEKHFClDghSpwQZZWyYc7OztZ9Ba6JlxOixAlR4oQocUKUOCFKnBAlToiy5+R/9vf3130F/uPlhChxQpQ4IUqcECVOiBInRIkTouw5N8yHDx9Wev6rV69Wej6/zssJUeKEKHFClDghSpwQJU6IEidE2XNumDdv3gzns9lsqfOfPXu21Oe5Ol5OiBInRIkTosQJUeKEKHFClDghyp4z5vz8fDj/+vXrcD6ZTJaa0+HlhChxQpQ4IUqcECVOiBInRFmlxLx79244//z581LnP378eDjf3t5e6nyujpcTosQJUeKEKHFClDghSpwQJU6IsueMOT4+Xun5BwcHw/mtW7dW+vfz67ycECVOiBInRIkTosQJUeKEKHFClD3nX+bly5frvgK/yMsJUeKEKHFClDghSpwQJU6IEidE2XNumNlsNpzfvHlzOL9///5VXocV8nJClDghSpwQJU6IEidEiROirFI2zGQyGc53d3eH8wcPHlzhbVglLydEiROixAlR4oQocUKUOCFKnBBlz/mH2d7eHs53dnau6SYsy8sJUeKEKHFClDghSpwQJU6IEidE2XPGLPvVldPp+Ee6tbW11PlcHy8nRIkTosQJUeKEKHFClDghSpwQNZnzK+XGv2+OK/ft27fh/OnTp8P5vF8BeHp6+tt3YuUu/DJiLydEiROixAlR4oQocUKUOCFKnBBlzwnrZ88Jm0ScECVOiBInRIkTosQJUeKEKHFClDghSpwQJU6IEidEiROixAlR4oQocUKUOCFKnBAlTogSJ0SJE6LECVHTOfMLv7IPWD0vJ0SJE6LECVHihChxQpQ4IeoHUlJeQ3HrW24AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([0., 1., 0., 0., 0., 0., 0., 0., 0., 0.])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1L9j3eNZkOVm"
      },
      "source": [
        "## Training\n",
        "\n",
        "Since MNIST is a 10-class dataset, we'll be performing multi-class classification which uses the cross entropy loss function.\n",
        "\n",
        "$$ -\\sum_{c=1}^My_{o,c}\\log(p_{o,c}) $$"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZQGoHzoBdkss"
      },
      "source": [
        "import torch\n",
        "\n",
        "from tqdm import trange\n",
        "\n",
        "def compute_multiclass_loss(Y, Y_hat):\n",
        "\n",
        "    L_sum = torch.sum(torch.multiply(Y, torch.log(Y_hat)))\n",
        "    m = Y.shape[1]\n",
        "    L = -(1/m) * L_sum\n",
        "\n",
        "    return L\n",
        "\n",
        "def sigmoid(z):\n",
        "    s = 1 / (1 + torch.exp(-z))\n",
        "    return s\n"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SGfvEvsAkvcD"
      },
      "source": [
        "Lets work with a two layer neural network with the hidden layer size being 64. We can go ahead and define them as shown below:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QlaYo_WodmVI"
      },
      "source": [
        "n_x = X_train.shape[0]\n",
        "n_h = 64\n",
        "learning_rate = 1\n",
        "\n",
        "W1 = torch.randn(n_h, n_x)\n",
        "b1 = torch.zeros((n_h, 1))\n",
        "W2 = torch.randn(digits, n_h)\n",
        "b2 = torch.zeros((digits, 1))"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KYedrkkglIaA"
      },
      "source": [
        "#### Training Loop\n",
        "\n",
        "Lets simply perform full batch gradient descent. This means passing the entire dataset through the model in one go"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 276,
          "referenced_widgets": [
            "5c919aedc1834306947697dd595adcf4",
            "0e4a7ec8a8fb4dc3812bc8b364db46e7",
            "5d2e65f38baa4b94a759c847ca828101",
            "68c6b3816ff04abab2cefa2a7d8b7718",
            "76ea284e541a4bbba1421a6cde343b00",
            "02c9ce06408d43b1b306550d99527783",
            "a77d1ba42abf43f783f26a944a5062b4",
            "c844af1092564f83aa9846ebad536ee3"
          ]
        },
        "id": "aQi3zf4Olf7p",
        "outputId": "761754bf-559b-4c4e-f362-84e0c2161973"
      },
      "source": [
        "\n",
        "X = torch.from_numpy(X_train).float()\n",
        "Y = torch.from_numpy(Y_train).float()\n",
        "\n",
        "\n",
        "from tqdm.notebook import tqdm\n",
        "pbar = tqdm()\n",
        "pbar.reset(total=100)\n",
        "\n",
        "for i in range(100):\n",
        "\n",
        "    ## Forward Pass\n",
        "\n",
        "    # Y = sigma(WX + b)\n",
        "    Z1 = torch.matmul(W1,X) + b1\n",
        "    A1 = sigmoid(Z1)\n",
        "\n",
        "    # Layer 2\n",
        "    Z2 = torch.matmul(W2,A1) + b2\n",
        "    \n",
        "    # Softmax\n",
        "    A2 = torch.exp(Z2) / torch.sum(np.exp(Z2), axis=0)\n",
        "\n",
        "    # Compute Loss\n",
        "    cost = compute_multiclass_loss(Y, A2)\n",
        "\n",
        "    ## Backward Pass\n",
        "\n",
        "    # Backpropagate to layer 2\n",
        "    dZ2 = A2-Y\n",
        "    dW2 = (1./m) * torch.matmul(dZ2, A1.T)\n",
        "    db2 = (1./m) * torch.sum(dZ2, axis=1, keepdims=True)\n",
        "\n",
        "    # Backpropagate to layer 1\n",
        "    dA1 = torch.matmul(W2.T, dZ2)\n",
        "    dZ1 = dA1 * sigmoid(Z1) * (1 - sigmoid(Z1))\n",
        "    dW1 = (1./m) * torch.matmul(dZ1, X.T)\n",
        "    db1 = (1./m) * torch.sum(dZ1, axis=1, keepdims=True)\n",
        "\n",
        "    # Update variables\n",
        "    W2 = W2 - learning_rate * dW2\n",
        "    b2 = b2 - learning_rate * db2\n",
        "    W1 = W1 - learning_rate * dW1\n",
        "    b1 = b1 - learning_rate * db1\n",
        "\n",
        "    pbar.set_description('Epoch %i' % i)\n",
        "    pbar.set_postfix(loss=cost.item())\n",
        "    pbar.update()\n",
        "\n",
        "    # if (i % 100 == 0):\n",
        "    #     print(\"Epoch\", i, \"cost: \", cost.item())\n",
        "\n",
        "print(\"Final Cost:\", cost.item())"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5c919aedc1834306947697dd595adcf4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-20-4c61220349e1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     35\u001b[0m     \u001b[0mdA1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mW2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdZ2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m     \u001b[0mdZ1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdA1\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0msigmoid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mZ1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0msigmoid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mZ1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 37\u001b[0;31m     \u001b[0mdW1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1.\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatmul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdZ1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mT\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     38\u001b[0m     \u001b[0mdb1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;36m1.\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mm\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdZ1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "63MQEcHAi600"
      },
      "source": [
        "## Testing our trained model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jcHm6FkHdsQ3",
        "outputId": "3da23404-a215-4782-c8e3-bbeac9e7edfc"
      },
      "source": [
        "from sklearn.metrics import classification_report, confusion_matrix\n",
        "\n",
        "X_test = torch.from_numpy(X_test).float()\n",
        "\n",
        "Z1 = torch.matmul(W1, X_test) + b1\n",
        "A1 = sigmoid(Z1)\n",
        "Z2 = torch.matmul(W2, A1) + b2\n",
        "A2 = torch.exp(Z2) / torch.sum(np.exp(Z2), axis=0)\n",
        "\n",
        "predictions = torch.argmax(A2, axis=0)\n",
        "labels = torch.argmax(torch.from_numpy(Y_test), axis=0)\n",
        "\n",
        "print(classification_report(predictions, labels))"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.88      0.85      0.87      1022\n",
            "           1       0.95      0.95      0.95      1144\n",
            "           2       0.76      0.78      0.77       997\n",
            "           3       0.77      0.76      0.76      1017\n",
            "           4       0.77      0.75      0.76      1004\n",
            "           5       0.63      0.65      0.64       855\n",
            "           6       0.83      0.84      0.84       945\n",
            "           7       0.79      0.83      0.81       986\n",
            "           8       0.70      0.70      0.70       975\n",
            "           9       0.73      0.70      0.71      1055\n",
            "\n",
            "    accuracy                           0.78     10000\n",
            "   macro avg       0.78      0.78      0.78     10000\n",
            "weighted avg       0.79      0.78      0.78     10000\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pECHxAr1jy2p"
      },
      "source": [
        "## Stochastic Gradient Descent"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lyCXFXM1l0T7"
      },
      "source": [
        "The difference between Stochatic Gradient Descent and Gradient Descent is that in SGD we do not pass the entire dataset at once. So we can define a function which returns us random splits"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jYCMADYnjD0v"
      },
      "source": [
        "def return_partitions(X, y, batch_size):\n",
        "    total_samples = X.shape[1]\n",
        "\n",
        "    assert total_samples % batch_size == 0\n",
        "    partitions = int(total_samples / batch_size)\n",
        "\n",
        "    return torch.split(X, batch_size, dim=1), torch.split(y, batch_size, dim=1)\n",
        "\n",
        "def partition(X, y, indices):\n",
        "    return X[:, indices], y[:, indices]"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PPVL_PM5jkta"
      },
      "source": [
        "px, py = return_partitions(torch.from_numpy(X_train).float(), torch.from_numpy(Y_train).float(), 200)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oIucIIgXmWZO"
      },
      "source": [
        "n_x = X_train.shape[0]\n",
        "n_h = 64\n",
        "learning_rate = 1\n",
        "\n",
        "W1 = torch.randn(n_h, n_x)\n",
        "b1 = torch.zeros((n_h, 1))\n",
        "W2 = torch.randn(digits, n_h)\n",
        "b2 = torch.zeros((digits, 1))"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "dc40c0293a2c463b8d5089f30201ee45",
            "36ecf0fa129e46e7a24c481dc1a60c99",
            "6b1fb292a81b40ab87d30c72fd10060b",
            "1eb4bccd2d134b08bb7002d8dbc00387",
            "d96c5bc52dd740bcb347fde0ab37ae71",
            "a9f3385b435941f2a7dd70becaf566c4",
            "801d55bc6aed4ed4a3f99303d545d63a",
            "0ca740c0f6e940339e8fd7f37ec1706c"
          ]
        },
        "id": "4n9OgC5xmW-J",
        "outputId": "fb2eb8f6-6865-4faa-87c3-701bbb9f25a2"
      },
      "source": [
        "from tqdm.notebook import tqdm\n",
        "pbar = tqdm()\n",
        "pbar.reset(total=100)\n",
        "\n",
        "for i in range(100):\n",
        "\n",
        "    ## Forward Pass\n",
        "\n",
        "    # Y = sigma(WX + b)\n",
        "    total_cost = 0.0\n",
        "    for tX, tY in zip(px, py):\n",
        "        Z1 = torch.matmul(W1,tX) + b1\n",
        "        A1 = sigmoid(Z1)\n",
        "\n",
        "        # Layer 2\n",
        "        Z2 = torch.matmul(W2,A1) + b2\n",
        "        \n",
        "        # Softmax\n",
        "        A2 = torch.exp(Z2) / torch.sum(np.exp(Z2), axis=0)\n",
        "\n",
        "        # Compute Loss\n",
        "        cost = compute_multiclass_loss(tY, A2)\n",
        "\n",
        "        ## Backward Pass\n",
        "\n",
        "        # Backpropagate to layer 2\n",
        "        dZ2 = A2-tY\n",
        "        dW2 = (1./m) * torch.matmul(dZ2, A1.T)\n",
        "        db2 = (1./m) * torch.sum(dZ2, axis=1, keepdims=True)\n",
        "\n",
        "        # Backpropagate to layer 1\n",
        "        dA1 = torch.matmul(W2.T, dZ2)\n",
        "        dZ1 = dA1 * sigmoid(Z1) * (1 - sigmoid(Z1))\n",
        "        dW1 = (1./m) * torch.matmul(dZ1, tX.T)\n",
        "        db1 = (1./m) * torch.sum(dZ1, axis=1, keepdims=True)\n",
        "\n",
        "        # Update variables\n",
        "        W2 = W2 - learning_rate * dW2\n",
        "        b2 = b2 - learning_rate * db2\n",
        "        W1 = W1 - learning_rate * dW1\n",
        "        b1 = b1 - learning_rate * db1\n",
        "        total_cost += cost.item()\n",
        "\n",
        "    pbar.set_description('Epoch %i' % i)\n",
        "    pbar.set_postfix(loss=total_cost/len(px))\n",
        "    pbar.update()\n",
        "\n",
        "print(\"Final Cost:\", cost.item())"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "dc40c0293a2c463b8d5089f30201ee45",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=1.0, bar_style='info', max=1.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Final Cost: 0.7561010718345642\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NQBClZxDpo3j",
        "outputId": "8fc1318c-9c3a-4a9b-b6e4-751673c00a3d"
      },
      "source": [
        "from sklearn.metrics import classification_report, confusion_matrix\n",
        "\n",
        "# X_test = torch.from_numpy(X_test).float()\n",
        "\n",
        "Z1 = torch.matmul(W1, X_test) + b1\n",
        "A1 = sigmoid(Z1)\n",
        "Z2 = torch.matmul(W2, A1) + b2\n",
        "A2 = torch.exp(Z2) / torch.sum(np.exp(Z2), axis=0)\n",
        "\n",
        "predictions = torch.argmax(A2, axis=0)\n",
        "labels = torch.argmax(torch.from_numpy(Y_test), axis=0)\n",
        "\n",
        "print(classification_report(predictions, labels))"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.88      0.85      0.86      1026\n",
            "           1       0.93      0.92      0.93      1146\n",
            "           2       0.75      0.81      0.78       960\n",
            "           3       0.78      0.72      0.75      1093\n",
            "           4       0.75      0.75      0.75       986\n",
            "           5       0.57      0.65      0.60       785\n",
            "           6       0.84      0.81      0.82       991\n",
            "           7       0.79      0.81      0.80      1007\n",
            "           8       0.65      0.64      0.65       982\n",
            "           9       0.71      0.70      0.71      1024\n",
            "\n",
            "    accuracy                           0.77     10000\n",
            "   macro avg       0.77      0.77      0.76     10000\n",
            "weighted avg       0.77      0.77      0.77     10000\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "u533DLz-o4pu"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}