code/06-radial-basis-function-networks.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "📝 **Author:** Amirhossein Heydari - 📧 **Email:** <amirhosseinheydari78@gmail.com> - 📍 **Origin:** [mr-pylin/pytorch-workshop](https://github.com/mr-pylin/pytorch-workshop)\n",
    "\n",
    "---\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Table of contents**<a id='toc0_'></a>    \n",
    "- [Dependencies](#toc1_)    \n",
    "- [Pre-Processing](#toc2_)    \n",
    "  - [Load Dataset](#toc2_1_)    \n",
    "  - [Split dataset into trainset & testset](#toc2_2_)    \n",
    "  - [Normalization](#toc2_3_)    \n",
    "  - [Dataset](#toc2_4_)    \n",
    "  - [Dataloader](#toc2_5_)    \n",
    "- [Radial Basis Function Networks](#toc3_)    \n",
    "- [Set up remaining Hyperparameters](#toc4_)    \n",
    "- [Train & Validation Loop](#toc5_)    \n",
    "- [Test Loop](#toc6_)    \n",
    "\n",
    "<!-- vscode-jupyter-toc-config\n",
    "\tnumbering=false\n",
    "\tanchor=true\n",
    "\tflat=false\n",
    "\tminLevel=1\n",
    "\tmaxLevel=6\n",
    "\t/vscode-jupyter-toc-config -->\n",
    "<!-- THIS CELL WILL BE REPLACED ON TOC UPDATE. DO NOT WRITE YOUR TEXT IN THIS CELL -->"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <a id='toc1_'></a>[Dependencies](#toc0_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from torch import nn\n",
    "from torch.optim import Adam\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "from torchinfo import summary\n",
    "from torchmetrics.classification import MulticlassAccuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set a seed for deterministic results\n",
    "seed = 0\n",
    "torch.manual_seed(seed)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.backends.cudnn.benchmark = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# check if cuda is available\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# log\n",
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <a id='toc2_'></a>[Pre-Processing](#toc0_)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <a id='toc2_1_'></a>[Load Dataset](#toc0_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate a 2D classification dataset\n",
    "n_samples = 250\n",
    "n_classes = 3\n",
    "\n",
    "X, y = make_classification(\n",
    "    n_samples=n_samples,\n",
    "    n_features=2,\n",
    "    n_informative=2,\n",
    "    n_redundant=0,\n",
    "    n_classes=n_classes,\n",
    "    n_clusters_per_class=1,\n",
    "    random_state=seed,\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <a id='toc2_2_'></a>[Split dataset into trainset & testset](#toc0_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <a id='toc2_3_'></a>[Normalization](#toc0_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = MinMaxScaler(feature_range=(-1, 1))\n",
    "scaler.fit(X_train)\n",
    "\n",
    "# log\n",
    "print(f\"min of trainset: {X_train.min(axis=0)}\")\n",
    "print(f\"max of trainset: {X_train.max(axis=0)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = scaler.transform(X_train)\n",
    "X_test = scaler.transform(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot\n",
    "plt.scatter(X_train[:, 0], X_train[:, 1], marker=\"o\", c=y_train, s=25, edgecolor=\"k\", label=\"trainset\")\n",
    "plt.scatter(X_test[:, 0], X_test[:, 1], marker=\",\", c=y_test, s=25, edgecolor=\"k\", label=\"testset\")\n",
    "plt.legend()\n",
    "plt.title(f\"2D dataset with {n_samples} samples\")\n",
    "plt.xlabel(\"feature 1\")\n",
    "plt.ylabel(\"feature 2\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <a id='toc2_4_'></a>[Dataset](#toc0_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = torch.tensor(X_train, dtype=torch.float32)\n",
    "X_test = torch.tensor(X_test, dtype=torch.float32)\n",
    "y_train = torch.tensor(y_train, dtype=torch.int64)\n",
    "y_test = torch.tensor(y_test, dtype=torch.int64)\n",
    "\n",
    "trainset = TensorDataset(X_train, y_train)\n",
    "testset = TensorDataset(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <a id='toc2_5_'></a>[Dataloader](#toc0_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 4\n",
    "\n",
    "trainloader = DataLoader(dataset=trainset, batch_size=batch_size, shuffle=True)\n",
    "testloader = DataLoader(dataset=testset, batch_size=batch_size, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <a id='toc3_'></a>[Radial Basis Function Networks](#toc0_)\n",
    "\n",
    "- Radial Basis Function (RBF) networks are indeed a type of artificial neural network.\n",
    "- They use radial basis functions as activation functions.\n",
    "- The output of the network is a linear combination of radial basis functions of the inputs and neuron parameters.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<figure style=\"text-align: center;\">\n",
    "  <img src=\"../assets/images/original/rbf/radial-basis-function-networks.svg\" alt=\"radial-basis-function-networks.svg\" style=\"width: 80%;\">\n",
    "  <figcaption>Radial Basis Functions Network Model</figcaption>\n",
    "</figure>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<table style=\"margin: 0 auto; text-align:center;\">\n",
    "  <thead>\n",
    "    <tr>\n",
    "      <th colspan=\"2\">hidden parameters</th>\n",
    "      <th colspan=\"2\">logits parameters</th>\n",
    "    </tr>\n",
    "  </thead>\n",
    "  <tbody>\n",
    "    <tr>\n",
    "      <td>μ</td>\n",
    "      <td>σ</td>\n",
    "      <td>Weights</td>\n",
    "      <td>Biases</td>\n",
    "    </tr>\n",
    "    <tr>\n",
    "      <td>A × B</td>\n",
    "      <td>B</td>\n",
    "      <td>B × C</td>\n",
    "      <td>C</td>\n",
    "    </tr>\n",
    "  </tbody>\n",
    "  <tfoot>\n",
    "    <tr>\n",
    "      <td colspan=\"2\">A × B + B</td>\n",
    "      <td colspan=\"2\">B × C + C</td>\n",
    "    </tr>\n",
    "  </tfoot>\n",
    "</table>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$\n",
    "X = \\begin{bmatrix}\n",
    "        x_{1}^1 & x_{1}^2 & \\cdots & x_{1}^N \\\\\n",
    "        x_{2}^1 & x_{2}^2 & \\cdots & x_{2}^N \\\\\n",
    "        \\vdots & \\vdots & \\ddots & \\vdots \\\\\n",
    "        x_{Q}^1 & x_{Q}^2 & \\cdots & x_{Q}^N \\\\\n",
    "    \\end{bmatrix}_{Q \\times N} \\quad \\text{(Q: number of samples, N: number of features)} \\\\\n",
    "$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$\n",
    "\\phi(X) = \n",
    "        % \\begin{bmatrix}\n",
    "        % 1 \\\\\n",
    "        % \\phi_{1} \\\\\n",
    "        % \\phi_{2} \\\\\n",
    "        % \\vdots \\\\\n",
    "        % \\phi_{N} \\\\\n",
    "        % \\end{bmatrix}_{(N+1) \\times 1}\n",
    "        \\begin{bmatrix}\n",
    "        1 & 1 & \\cdots & 1 \\\\\n",
    "        \\phi_1(X_1) & \\phi_1(X_2) & \\cdots & \\phi_1(X_Q) \\\\\n",
    "        \\phi_2(X_1) & \\phi_2(X_2) & \\cdots & \\phi_2(X_Q) \\\\\n",
    "        \\vdots & \\vdots & \\ddots & \\vdots \\\\\n",
    "        \\phi_N(X_1) & \\phi_N(X_2) & \\cdots & \\phi_N(X_Q) \\\\\n",
    "        \\end{bmatrix}_{(N+1) \\times Q}\n",
    "    = \n",
    "        \\begin{bmatrix}\n",
    "        1 & 1 & \\cdots & 1 \\\\\n",
    "        \\exp\\left(-\\frac{\\|\\mathbf{X_1} - \\mathbf{\\mu}_1\\|^2}{2\\sigma_1^2}\\right) & \\exp\\left(-\\frac{\\|\\mathbf{X_2} - \\mathbf{\\mu}_1\\|^2}{2\\sigma_1^2}\\right) & \\cdots & \\exp\\left(-\\frac{\\|\\mathbf{X_Q} - \\mathbf{\\mu}_1\\|^2}{2\\sigma_1^2}\\right) \\\\\n",
    "        \\exp\\left(-\\frac{\\|\\mathbf{X_1} - \\mathbf{\\mu}_2\\|^2}{2\\sigma_2^2}\\right) & \\exp\\left(-\\frac{\\|\\mathbf{X_2} - \\mathbf{\\mu}_2\\|^2}{2\\sigma_2^2}\\right) & \\cdots & \\exp\\left(-\\frac{\\|\\mathbf{X_Q} - \\mathbf{\\mu}_2\\|^2}{2\\sigma_2^2}\\right) \\\\\n",
    "        \\vdots & \\vdots & \\ddots & \\vdots \\\\\n",
    "        \\exp\\left(-\\frac{\\|\\mathbf{X_1} - \\mathbf{\\mu}_N\\|^2}{2\\sigma_N^2}\\right) & \\exp\\left(-\\frac{\\|\\mathbf{X_2} - \\mathbf{\\mu}_N\\|^2}{2\\sigma_N^2}\\right) & \\cdots & \\exp\\left(-\\frac{\\|\\mathbf{X_Q} - \\mathbf{\\mu}_N\\|^2}{2\\sigma_N^2}\\right) \\\\\n",
    "        \\end{bmatrix}_{(N+1) \\times Q}\n",
    "$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$\n",
    "W = \\begin{bmatrix}\n",
    "        w_{0}^1 & w_{0}^2 & \\cdots & w_{0}^{C} \\\\\n",
    "        w_{1}^1 & w_{1}^2 & \\cdots & w_{1}^{C} \\\\\n",
    "        w_{2}^1 & w_{2}^2 & \\cdots & w_{2}^{C} \\\\\n",
    "        \\vdots & \\vdots & \\ddots & \\vdots \\\\\n",
    "        w_{N}^1 & w_{N}^2 & \\cdots & w_{N}^{C} \\\\\n",
    "    \\end{bmatrix}_{(N+1) \\times C} \\quad \\text{(N: number of kernels, C: number of output neurons)}\n",
    "$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$f(X) = \\phi(X)^TW$$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class RBF(nn.Module):\n",
    "    def __init__(self, in_features: int, out_features: int, kernel: str = \"gaussian\"):\n",
    "        super().__init__()\n",
    "        self.in_features = in_features\n",
    "        self.out_features = out_features\n",
    "        self.centers = nn.Parameter(torch.Tensor(out_features, in_features))\n",
    "        self.sigmas = nn.Parameter(torch.Tensor(out_features))\n",
    "        self.kernel = kernel\n",
    "        self.reset_parameters()\n",
    "\n",
    "    def reset_parameters(self) -> None:\n",
    "        nn.init.uniform_(self.centers, -1.0, 1.0)\n",
    "        nn.init.uniform_(self.sigmas, 0.0, 1.0)\n",
    "\n",
    "    def forward(self, input: torch.Tensor) -> torch.Tensor:\n",
    "        size = (input.size(0), self.out_features, self.in_features)\n",
    "        x = input.unsqueeze(1).expand(size)\n",
    "        c = self.centers.unsqueeze(0).expand(size)\n",
    "        distances = (x - c).pow(2).sum(-1).pow(0.5)\n",
    "\n",
    "        if self.kernel == \"gaussian\":\n",
    "            return torch.exp(-1.0 * distances.pow(2) / (2 * self.sigmas.unsqueeze(0) ** 2))\n",
    "\n",
    "        elif self.kernel == \"linear\":\n",
    "            return 1.0 - distances\n",
    "\n",
    "        else:\n",
    "            raise ValueError(f\"Unknown kernel type {self.kernel}\")\n",
    "\n",
    "\n",
    "class RBFNet(nn.Module):\n",
    "    def __init__(self, in_features: int, hidden_features: int, out_features: int, kernel: str = \"gaussian\"):\n",
    "        super().__init__()\n",
    "        self.rbf = RBF(in_features, hidden_features, kernel)\n",
    "        self.linear = nn.Linear(hidden_features, out_features)\n",
    "\n",
    "    def forward(self, input: torch.Tensor) -> torch.Tensor:\n",
    "        return self.linear(self.rbf(input))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "in_features = trainset[0][0].shape[0]\n",
    "num_kernels = 3\n",
    "out_features = n_classes\n",
    "\n",
    "# initialize the model\n",
    "model = RBFNet(in_features, num_kernels, out_features).to(device)\n",
    "\n",
    "# log\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "summary(model, input_size=(batch_size, in_features))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <a id='toc4_'></a>[Set up remaining Hyperparameters](#toc0_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = 0.01\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = Adam(params=model.parameters(), lr=lr)\n",
    "num_epochs = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <a id='toc5_'></a>[Train & Validation Loop](#toc0_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# plot\n",
    "def plot():\n",
    "    centers = model.rbf.centers.detach().cpu().numpy()\n",
    "    radii = model.rbf.sigmas.detach().cpu().numpy()\n",
    "\n",
    "    for center, radius in zip(centers, radii):\n",
    "        circle = plt.Circle(center, radius, color=\"b\", fill=False)\n",
    "        plt.gca().add_artist(circle)\n",
    "\n",
    "    plt.scatter(X_train[:, 0], X_train[:, 1], marker=\"o\", c=y_train, s=25, edgecolor=\"k\", label=\"trainset\")\n",
    "    plt.scatter(X_test[:, 0], X_test[:, 1], marker=\",\", c=y_test, s=25, edgecolor=\"k\", label=\"testset\")\n",
    "    plt.legend()\n",
    "    plt.title(f\"2D dataset with {n_samples} samples\")\n",
    "    plt.xlabel(\"feature 1\")\n",
    "    plt.ylabel(\"feature 2\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_acc_per_epoch = []\n",
    "train_loss_per_epoch = []\n",
    "\n",
    "train_acc = MulticlassAccuracy(num_classes=n_classes, top_k=1).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for epoch in range(num_epochs):\n",
    "\n",
    "    # train loop\n",
    "    model.train()\n",
    "    train_loss = 0\n",
    "\n",
    "    for x, y in trainloader:\n",
    "\n",
    "        # send data to GPU\n",
    "        x, y_true = x.to(device), y.to(device)\n",
    "\n",
    "        # forward\n",
    "        y_pred = model(x)\n",
    "        loss = criterion(y_pred, y_true)\n",
    "\n",
    "        # backward\n",
    "        loss.backward()\n",
    "\n",
    "        # update parameters\n",
    "        optimizer.step()\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        # store loss and accuracy per iteration\n",
    "        train_loss += loss.item() * len(x)\n",
    "        train_acc.update(y_pred, y_true)\n",
    "\n",
    "    # store loss and accuracy per epoch\n",
    "    train_loss_per_epoch.append(train_loss / len(trainset))\n",
    "    train_acc_per_epoch.append(train_acc.compute().item())\n",
    "    train_acc.reset()\n",
    "\n",
    "    # log\n",
    "    print(\n",
    "        f\"epoch {epoch+1:0{len(str(num_epochs))}}/{num_epochs} -> train[loss: {train_loss_per_epoch[epoch]:7.5f} - acc: {train_acc_per_epoch[epoch]*100:5.2f}%]\"\n",
    "    )\n",
    "\n",
    "    # plot\n",
    "    plot()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <a id='toc6_'></a>[Test Loop](#toc0_)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_acc = MulticlassAccuracy(num_classes=n_classes, top_k=1).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test loop\n",
    "model.eval()\n",
    "test_loss = 0\n",
    "predictions = []\n",
    "targets = []\n",
    "\n",
    "with torch.no_grad():\n",
    "    for x, y in testloader:\n",
    "\n",
    "        # send data to GPU\n",
    "        x, y_true = x.to(device), y.to(device)\n",
    "\n",
    "        # forward\n",
    "        y_pred = model(x)\n",
    "        loss = criterion(y_pred, y_true)\n",
    "\n",
    "        # store loss and accuracy per iteration\n",
    "        test_loss += loss.item() * len(x)\n",
    "        test_acc.update(y_pred, y_true)\n",
    "\n",
    "        predictions.extend(y_pred.argmax(dim=1).cpu())\n",
    "        targets.extend(y_true.cpu())\n",
    "\n",
    "# log\n",
    "print(f\"test[loss: {test_loss / len(testset):.5f} - acc: {test_acc.compute().item()*100:5.2f}%]\")"
   ]
  }
 ],
 "metadata": {
  "author_email": "AmirhosseinHeydari78@gmail.com",
  "author_github": "https://github.com/mr-pylin",
  "author_name": "Amirhossein Heydari",
  "kernelspec": {
   "display_name": "pytorch-workshop-U_zYfVTd-py3.12",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.8"
  },
  "origin_repo": "https://github.com/mr-pylin/pytorch-workshop"
 },
 "nbformat": 4,
 "nbformat_minor": 2
}