04_mnist.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python [conda env:ai-notebooks] *",
      "language": "python",
      "name": "conda-env-ai-notebooks-py"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.1"
    },
    "colab": {
      "name": "fastai_mnist.ipynb",
      "provenance": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "t_kkM7VebmnD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "214f98e3-4661-4e52-b6a2-973f20935236"
      },
      "source": [
        "!pip install -Uqq fastbook\n",
        "import fastbook \n",
        "fastbook.setup_book()"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 727kB 5.5MB/s \n",
            "\u001b[K     |████████████████████████████████| 194kB 17.0MB/s \n",
            "\u001b[K     |████████████████████████████████| 51kB 5.3MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.2MB 17.8MB/s \n",
            "\u001b[K     |████████████████████████████████| 61kB 6.1MB/s \n",
            "\u001b[K     |████████████████████████████████| 51kB 6.1MB/s \n",
            "\u001b[?25hMounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CYXRq6ObcT5o"
      },
      "source": [
        "from fastai.vision.all import *\n",
        "from fastbook import *\n",
        "from time import time"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3I3XYIEktkpU"
      },
      "source": [
        "torch.random.manual_seed(42);\n",
        "torch.set_printoptions(sci_mode=False)"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OKfRGiAG81pC"
      },
      "source": [
        "## Load data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kZ6FPr1Gcmyx"
      },
      "source": [
        "path = untar_data(URLs.MNIST)\n",
        "Path.BASE_PATH = path"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ElpnIcsIUz_n"
      },
      "source": [
        "digits = DataBlock(blocks=(ImageBlock(cls=PILImageBW), CategoryBlock),\n",
        "                   get_items=get_image_files,\n",
        "                   splitter=GrandparentSplitter(train_name='training', valid_name='testing'),\n",
        "                   get_y=parent_label)"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q0LuCWpBU1dz",
        "outputId": "ef963c29-fce5-4fed-f7a9-5d9990fc69bd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "dls = digits.dataloaders(path)\n",
        "print(dls.valid.one_batch()[0][0].shape)"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([1, 28, 28])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pEU1AsWA-F5V"
      },
      "source": [
        "## Loss function"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GSIwom8blbBV"
      },
      "source": [
        "def softmax(x): \n",
        "  return torch.exp(x) / torch.exp(x).sum(dim=1, keepdim=True)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "h_4Ibw1UwBS7"
      },
      "source": [
        "The torch [docs](https://pytorch.org/docs/stable/nn.functional.html#log-softmax) explain that in practise doing softmax() followed by log() is slower and numerically unstable. Note to myself to checkout F.log_softmax() to see what they are doing differently when I'm further into the course."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pddw_87alaYR"
      },
      "source": [
        "def neg_log_likelihood(x, targ):\n",
        "  return (-x[range(len(targ)), targ])"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Zi_KW43nCqz"
      },
      "source": [
        "def cross_entropy_loss(acts, targ, reduction=\"mean\"):\n",
        "  preds = torch.log(softmax(acts))\n",
        "  return neg_log_likelihood(preds, targ).mean() if reduction=='mean' else neg_log_likelihood(preds, targ)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9eKg1ZQ6Sfh3"
      },
      "source": [
        "## Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-QXlKWgKj8nv"
      },
      "source": [
        "def batch_accuracy(preds, yb):\n",
        "  preds = preds.argmax(dim=1)\n",
        "  correct = preds == yb\n",
        "  return correct.float().mean()"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gv7AMiXekGOY"
      },
      "source": [
        "class BasicOptimiser:\n",
        "  def __init__(self, params, lr):\n",
        "    self.params,self.lr = list(params),lr\n",
        "\n",
        "  def step(self):\n",
        "    for p in self.params:\n",
        "      p.data -= p.grad.data * self.lr\n",
        "\n",
        "  def zero_grad(self):\n",
        "    for p in self.params:\n",
        "      p.grad = None"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-oFPB37skB3R"
      },
      "source": [
        "class BasicLearner:\n",
        "  def __init__(self, dls: DataLoaders, model, opt_func, loss_function, batch_accuracy):\n",
        "    self.dls = dls\n",
        "    self.model = model\n",
        "    self.opt_func = opt_func(model.parameters(), 0.03)\n",
        "    self.loss_function = loss_function\n",
        "    self.batch_accuracy = batch_accuracy\n",
        "\n",
        "  def validate_epoch(self):\n",
        "    accs = [self.batch_accuracy(self.model(xb), yb) for xb, yb in self.dls.valid]\n",
        "    return round(torch.stack(accs).mean().item(), 4)\n",
        "\n",
        "  def fit(self, epochs):\n",
        "    for epoch in range(epochs):\n",
        "      accs = []\n",
        "      start_time = time()\n",
        "      for xb, yb in self.dls.train:\n",
        "        preds = self.model(xb)\n",
        "        loss = self.loss_function(preds, yb)\n",
        "        loss.backward()\n",
        "        self.opt_func.step()\n",
        "        self.opt_func.zero_grad()\n",
        "      print(f\"Epoch {epoch}, Accuracy: {self.validate_epoch()}, took {time() - start_time:.2f}s\")\n",
        "\n",
        "  def pred(self, xb):\n",
        "    return self.model(xb).argmax(dim=1)"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bu_jjl5lqfzI"
      },
      "source": [
        "simple_net = nn.Sequential(\n",
        "    nn.Flatten(),\n",
        "    nn.Linear(28*28,30),\n",
        "    nn.ReLU(),\n",
        "    nn.Linear(30,10)\n",
        ")\n",
        "simple_net.to(torch.cuda.current_device())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FhcXL7q7sMiO",
        "outputId": "88152f76-3538-4678-c3e4-19ee54397de7"
      },
      "source": [
        "learner = BasicLearner(dls, simple_net, BasicOptimiser, cross_entropy_loss, batch_accuracy)\n",
        "learner.fit(10)"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 0, Accuracy: 0.9011, took 72.18s\n",
            "Epoch 1, Accuracy: 0.9135, took 71.68s\n",
            "Epoch 2, Accuracy: 0.9241, took 71.44s\n",
            "Epoch 3, Accuracy: 0.9288, took 71.52s\n",
            "Epoch 4, Accuracy: 0.9341, took 71.67s\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PHS0LUnYVccd"
      },
      "source": [
        "## Experiment: Using median loss"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "g0Stj2htVy4L"
      },
      "source": [
        "In the lecture someone asked why you wouldn't use median loss, so I decided to try it out."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dkt1CQmPw9fr"
      },
      "source": [
        "def mnist_loss_median(preds, target):\n",
        "  return torch.where(target==1, 1-preds, preds).median()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sQ2H2XLXViZx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "26be82f8-aaa1-49ad-e0b7-9da25cb31cc9"
      },
      "source": [
        "model = SimpleNet(28*28, 1)\n",
        "learner_exp1 = BasicLearner(dls, model, BasicOptimiser(model.parameters(), 0.13), mnist_loss_median, batch_accuracy)\n",
        "learner_exp1.fit(10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 0, Accuracy: 0.5068\n",
            "Epoch 1, Accuracy: 0.5068\n",
            "Epoch 2, Accuracy: 0.5068\n",
            "Epoch 3, Accuracy: 0.5068\n",
            "Epoch 4, Accuracy: 0.5166\n",
            "Epoch 5, Accuracy: 0.5552\n",
            "Epoch 6, Accuracy: 0.6074\n",
            "Epoch 7, Accuracy: 0.6519\n",
            "Epoch 8, Accuracy: 0.7769\n",
            "Epoch 9, Accuracy: 0.8369\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ia9qnz4kVone"
      },
      "source": [
        "It seems that even though it converges at some point in this case, it just trains slower - don't have a strong intuition for why that is yet."
      ]
    }
  ]
}