MINST_Optimizer.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "T1VrgP2-1C4M"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.nn.functional as F\n",
        "from torchvision.datasets import MNIST\n",
        "from torchvision import transforms,datasets\n",
        "from torch.utils.data import DataLoader,random_split\n",
        "\n",
        "import numpy as np\n",
        "from matplotlib import pyplot as plt\n",
        "from torchsummary import summary"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Fix random seed"
      ],
      "metadata": {
        "id": "K6JU0oKjYqqH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import random\n",
        "# Set the seed for Python's built-in random module\n",
        "random_seed = 42\n",
        "random.seed(random_seed)\n",
        "\n",
        "# Set the seed for NumPy\n",
        "np.random.seed(random_seed)\n",
        "\n",
        "# Set the seed for PyTorch\n",
        "torch.manual_seed(random_seed)\n",
        "torch.cuda.manual_seed_all(random_seed)\n",
        "torch.backends.cudnn.deterministic = True\n",
        "torch.backends.cudnn.benchmark = False"
      ],
      "metadata": {
        "id": "T4SCyh49Yp3v"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S2Ht0iPM1C4P"
      },
      "source": [
        "### Loading Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We can load MNIST from Pytroch directly"
      ],
      "metadata": {
        "id": "Z5smG27qSG7f"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from torchvision.datasets import MNIST"
      ],
      "metadata": {
        "id": "0ttiTtLCR8Su"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Data Preprocessing"
      ],
      "metadata": {
        "id": "Z3oTzpZBSC5_"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "For better performance, we will standardize the input tensor (or images) using predetermined mean and standard deviation values (mean=0.1307, sd=0.3081)\n",
        "\n",
        "\n",
        "Subsequently, we'll follow the usal procedure of splitting the dataset into training, validation, and testing sets."
      ],
      "metadata": {
        "id": "sFZNvx4ASM6X"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size_train = 64\n",
        "batch_size_test = 1000\n",
        "\n",
        "transform=transforms.Compose([\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize((0.1307,), (0.3081,))\n",
        "    ])\n",
        "\n",
        "train_loader = torch.utils.data.DataLoader(\n",
        "    MNIST('/files/', train=True, download=True,\n",
        "                             transform = transform),\n",
        "  batch_size=batch_size_train, shuffle=True)\n",
        "\n",
        "\n",
        "test_loader = torch.utils.data.DataLoader(\n",
        "    MNIST('/files/', train=False, download=True,\n",
        "                             transform = transform),\n",
        "  batch_size=batch_size_test, shuffle=True)"
      ],
      "metadata": {
        "id": "7yx6EMxCS0-c"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "The shape of inputs for tranining would be batch size = 64   \n",
        "channel = 1   \n",
        "height,width = 28, 28"
      ],
      "metadata": {
        "id": "8J7Re-g5XBYv"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8VTIGBJk1C4Q",
        "outputId": "fd6c1124-9c93-4596-82c1-cce6d3793788"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([64, 1, 28, 28])"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "examples = enumerate(train_loader)\n",
        "batch_idx, (example_data, example_targets) = next(examples)\n",
        "example_data.shape"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Let's display some picture for the dataset"
      ],
      "metadata": {
        "id": "fzsDCRiJXXNO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "fig = plt.figure()\n",
        "for i in range(3):\n",
        "  plt.subplot(2,3,i+1)\n",
        "  plt.tight_layout()\n",
        "  plt.imshow(example_data[i][0], cmap='gray', interpolation='none')\n",
        "  plt.title(\"Label: {}\".format(example_targets[i]))\n",
        "  plt.xticks([])\n",
        "  plt.yticks([])\n",
        "fig"
      ],
      "metadata": {
        "id": "i-fby73vXaH6",
        "outputId": "6b8aa87f-d35b-4a67-b5f9-0b501aa64a48",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 435
        }
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 3 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmYAAADRCAYAAABmfH5uAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAWpUlEQVR4nO3df1BVdf7H8fcVFUFRAkTzRyhrppuaLYw/GiksTR1ZR3eVrdWwsnLKWnMwtJkMW7dtTN12EEdXM3PTnFxGXHOtZiydkVK0nOiXWqCsmuVKKlx/AcrZP5r4rl/fp8vBc7mfc3k+Zvqjl/d+zhvmfpyXRz4en2VZlgAAACDkWoR6AAAAAPyIYgYAAGAIihkAAIAhKGYAAACGoJgBAAAYgmIGAABgCIoZAACAIShmAAAAhqCYAQAAGIJidp3Ky8vF5/PJ4sWLXVtz586d4vP5ZOfOna6tCXgJ+wpwH/vKG5plMXv99dfF5/PJxx9/HOpRguLQoUMya9YsueOOO6RNmzbi8/mkvLw81GMhzIX7vtq0aZP87ne/k+TkZImOjpZbbrlFsrOz5ezZs6EeDWEs3PeViMj27dtl+PDhkpCQILGxsTJo0CB54403Qj1WyDTLYhbudu/eLXl5eeL3+6Vv376hHgcIC4899pgcOHBApkyZInl5eTJ69GjJz8+XoUOHysWLF0M9HuBJW7ZskXvvvVdqampk/vz58uKLL0pUVJRkZWXJK6+8EurxQqJlqAeA+8aNGydnz56VmJgYWbx4sXz66aehHgnwvIKCAklPT78qS0lJkalTp8r69evlkUceCc1ggIfl5+fLjTfeKB988IFERkaKiMj06dOlT58+8vrrr8usWbNCPGHT446ZjZqaGnn++eclJSVFOnToIG3btpW0tDTZsWOH7XteeeUVSUpKkqioKLnrrrvkiy++uOY1Bw8elIkTJ0pcXJy0adNGUlNTZcuWLQHnuXDhghw8eFAqKioCvjYuLk5iYmICvg5oal7eV/+/lImITJgwQUREDhw4EPD9QLB4eV9VVVXJDTfcUF/KRERatmwpCQkJEhUVFfD94YhiZqOqqkpeffVVSU9Pl4ULF8r8+fPl1KlTMmrUKPUO1N///nfJy8uTGTNmyLPPPitffPGF3H333XLy5Mn613z55ZcyZMgQOXDggMydO1eWLFkibdu2lfHjx0thYeHPzrN3717p27ev5Ofnu/2lAk0m3PbV999/LyIiCQkJjXo/4AYv76v09HT58ssvZd68eVJaWiplZWWyYMEC+fjjjyUnJ8fx9yIsWM3QmjVrLBGx9u3bZ/uay5cvW9XV1VdlZ86csTp16mQ9/PDD9dmRI0csEbGioqKs48eP1+fFxcWWiFizZs2qz+655x6rf//+1qVLl+qzuro664477rBuvvnm+mzHjh2WiFg7duy4JsvNzXX0tS5atMgSEevIkSOO3gc41Zz21U+mTZtmRUREWF9//XWj3g8EEu776ty5c1ZmZqbl8/ksEbFExIqOjrY2b94c8L3hijtmNiIiIqR169YiIlJXVyenT5+Wy5cvS2pqquzfv/+a148fP166du1a//+DBg2SwYMHy7Zt20RE5PTp0/LBBx9IZmam+P1+qaiokIqKCvnhhx9k1KhR8s0338i3335rO096erpYliXz58939wsFmlA47as333xTVq9eLdnZ2XLzzTc7fj/gFi/vq8jISOndu7dMnDhRNmzYIOvWrZPU1FSZMmWK7Nmzx+F3Ijzww/8/Y+3atbJkyRI5ePCg1NbW1uc9e/a85rXab8y9e/eWjRs3iohIaWmpWJYl8+bNk3nz5qnX+89//nPVZgHCUTjsq127dsm0adNk1KhR8uKLL7q6NtAYXt1XTz75pOzZs0f2798vLVr8eK8oMzNTbr31Vpk5c6YUFxdf9zW8hmJmY926dfLggw/K+PHj5ZlnnpHExESJiIiQl156ScrKyhyvV1dXJyIis2fPllGjRqmv6dWr13XNDJguHPZVSUmJjBs3Tvr16ycFBQXSsiW/jSK0vLqvampqZPXq1ZKTk1NfykREWrVqJWPGjJH8/HypqampvxvYXPA7io2CggJJTk6WTZs2ic/nq89zc3PV13/zzTfXZF9//bX06NFDRESSk5NF5McP3IgRI9wfGPAAr++rsrIyGT16tCQmJsq2bdukXbt2Qb8mEIhX99UPP/wgly9flitXrlzza7W1tVJXV6f+WrjjZ8xsREREiIiIZVn1WXFxsezevVt9/ebNm6/6O/e9e/dKcXGxjBkzRkREEhMTJT09Xf72t7/Jd999d837T5069bPzODl+DJjKy/vq+++/l3vvvVdatGgh7733nnTs2DHge4Cm4NV9lZiYKLGxsVJYWCg1NTX1+blz5+Ttt9+WPn36NMt/MqNZ3zF77bXX5N13370mnzlzpmRkZMimTZtkwoQJMnbsWDly5IisWLFCfvnLX8q5c+eueU+vXr1k2LBh8vjjj0t1dbX89a9/lfj4+KuO+y5btkyGDRsm/fv3l0cffVSSk5Pl5MmTsnv3bjl+/LiUlJTYzrp3714ZPny45ObmBvyBysrKSlm6dKmIiHz44Yci8uM/4hcbGyuxsbHy5JNPNuTbAzRKuO6r0aNHy+HDhyUnJ0eKioqkqKio/tc6deokI0eObMB3B2iccNxXERERMnv2bHnuuedkyJAhkpWVJVeuXJHVq1fL8ePHZd26dc6+SeEiZOdBQ+in48d2/x07dsyqq6uz/vznP1tJSUlWZGSkdfvtt1tbt261pk6daiUlJdWv9dPx40WLFllLliyxunfvbkVGRlppaWlWSUnJNdcuKyuzsrKyrM6dO1utWrWyunbtamVkZFgFBQX1r7ne48c/zaT997+zA24K9331c1/bXXfddR3fOcBeuO8ry7Ks9evXW4MGDbJiY2OtqKgoa/DgwVddo7nxWdb/3PsEAABAyPAzZgAAAIagmAEAABiCYgYAAGAIihkAAIAhKGYAAACGoJgBAAAYokH/wGxdXZ2cOHFCYmJirnrcAxBqlmWJ3++XLl26XPWsNS9gX8FU7CvAfQ3dVw0qZidOnJDu3bu7NhzgtmPHjkm3bt1CPYYj7CuYjn0FuC/QvmrQH4ViYmJcGwgIBi9+Rr04M5oXL35GvTgzmpdAn9EGFTNuB8N0XvyMenFmNC9e/Ix6cWY0L4E+o9764QEAAIAwRjEDAAAwBMUMAADAEBQzAAAAQ1DMAAAADEExAwAAMATFDAAAwBAUMwAAAENQzAAAAAxBMQMAADAExQwAAMAQFDMAAABDUMwAAAAMQTEDAAAwBMUMAADAEBQzAAAAQ1DMAAAADEExAwAAMATFDAAAwBAUMwAAAEO0DPUAaFoDBw5U8wULFqh5RkaGmsfHx6v56dOnGzUXvCk2NlbNH3nkETVfunSpmnfs2FHNc3NzHa3vVGlpqZrfc889av7dd9+peW1trSvzAAB3zAAAAAxBMQMAADAExQwAAMAQFDMAAABDUMwAAAAMwanMMNWqVSs1z87OVvOxY8eqeU1NjZpbltW4weBJPp9PzRctWqTm06ZNU/OXX37ZlXnsPn9XrlxR84sXL6p5ly5d1NzutGZhYaGa33///WpeV1en5sDPiYyMVPPWrVureadOndR86tSpjq5rd9o5MTHR0Tp2tm/frua/+c1v1Pz8+fOuXNdruGMGAABgCIoZAACAIShmAAAAhqCYAQAAGIJiBgAAYAhOZXqc3SmdVatWqfnkyZPVvLq6Ws0zMzPV/MyZMw2YDuHivvvuU3O705du8fv9am73bMqFCxequd3p0R49eqh5v3791Nzu2Z12r//ss8/UHBARSUpKUnO7078DBgwI5ji23DqFP2LECDW3ewbthAkT1Pz99993ZR5TcccMAADAEBQzAAAAQ1DMAAAADEExAwAAMATFDAAAwBCcyvQ4u9NyDzzwgKN1VqxYoeZvv/2245ngXW3atFHzuXPnOlrn6NGjav7RRx+p+a5du9T8nXfeUfPy8nJH89ixW8cub9eunZo/99xzam73DE27Z3qieTl8+LCaN7dnEUdHR6v5ypUr1fy2225T83Pnzrk2UyhxxwwAAMAQFDMAAABDUMwAAAAMQTEDAAAwBMUMAADAEJzK9Ihhw4ap+ZIlSxytc/78eTVfvHix45kQfuxOC+bl5Tlap6ioSM0PHTrkeCYvmDhxoprPnj1bze1OrQKNcfHiRTV3ut/sTk1nZGSo+U033eRofac6d+6s5kOGDFHz7du3q3nv3r3VPCUlRc3Hjx+v5u+++66ar1mzRs0biztmAAAAhqCYAQAAGIJiBgAAYAiKGQAAgCEoZgAAAIbgVKZh2rZtq+b5+flqHh8fr+Z+v1/NJ0+erObffvttA6ZDuKutrVXz1atXN/Ek4SEuLk7NOZUJNy1YsEDNX375ZVfWt1unX79+ap6ZmanmU6dOdXTdLVu2qHlpaama2z2b1u5U+Q033OBontTUVDXfsGGDml+6dMnR+j/hjhkAAIAhKGYAAACGoJgBAAAYgmIGAABgCIoZAACAITiVGSJ2py+XL1+u5gMGDFDzqqoqNc/KylLzrVu3NmA6AG6YMWOGmj/66KNNPAnC2dChQ4O6/vHjxx3lv/3tb1257qRJk9Tc7uvt1q2bK9e1k5SUpOZjxoxR88LCwkZdhztmAAAAhqCYAQAAGIJiBgAAYAiKGQAAgCEoZgAAAIbgVGaIZGRkqPmUKVMcrbN+/Xo1t3vGGIDG69y5s6PXb9y4MUiTIBy89tprav7QQw85WmfEiBFqPmfOHDX/8MMP1byoqMjRde2eTWn3TEy7Z/GWlJSoeUpKipoH+/SlU//4xz/UvGXLxlUs7pgBAAAYgmIGAABgCIoZAACAIShmAAAAhqCYAQAAGMJnWZYV6EVVVVXSoUOHppgn7Lzwwgtq/sQTT6h5fHy8mm/evFnN7U7vVFZWBh4ujFRWVkr79u1DPYYj7CtzxcXFqfm+ffvUvGfPnmreo0cPNT969Gij5mpq7Kvg6t69u5ofOXIkqNe9cOGCmts9w/Wtt95S888//1zN+/btq+Y+n0/NG1BDrsvSpUvVfNmyZWo+ffp0NY+NjXV0XbvvZ6B9xR0zAAAAQ1DMAAAADEExAwAAMATFDAAAwBAUMwAAAENwKtMlds8q27Bhg5rbnb48c+aMmqelpan5V1991YDpwh+nx+Cmp59+Ws3/8pe/qPnJkyfVfODAgY5ebxr2VXBFRkaqud0pwocffjiY48jFixfVvKqqSs07duyo5i1a6Pd8gn0qs7i4WM2HDx+u5jU1Na5c1ylOZQIAAHgExQwAAMAQFDMAAABDUMwAAAAMQTEDAAAwRMtQD+A1ycnJau7W6cusrCw15/Ql4D67U2UzZsxwtM6KFSvU3CunLxEa1dXVap6dna3mqampaj5gwABX5omKinKUh8ry5cvVPCcnR81DdfqysbhjBgAAYAiKGQAAgCEoZgAAAIagmAEAABiCYgYAAGAITmXasHvG3Zw5c9Tc7vTlvn371PyPf/yjmv/rX/8KPBzQxFq3bq3mEyZMUPNZs2Y5Wn/Xrl1q3qtXLzV/9dVX1XzKlClq3rNnTzWPjY1V81/84hdqbufEiRNqPnbsWEfrOFVaWqrmhw4dCup1EVx+v1/Nf/WrX6m53SnivLw812Zyg90zNCsqKtR827Ztav7UU0+5NpOJuGMGAABgCIoZAACAIShmAAAAhqCYAQAAGIJiBgAAYAifZVlWoBdVVVVJhw4dmmKeJhcdHa3mdqdB7rzzTjU/e/asmo8cOVLNP/nkk8DDocEqKyulffv2oR7DERP3ld2przVr1qh5//79gzkOAvjTn/6k5s8//7wr67OvvGHYsGFqvnPnzqYdJACfz6fmM2fOVPP8/PxgjhMygfYVd8wAAAAMQTEDAAAwBMUMAADAEBQzAAAAQ1DMAAAADNFsnpUZFRWl5nanzexOX1ZWVqr55MmT1ZzTlzBRRESEmufm5qq509OX58+fV/OtW7c6Wsep9PR0Ne/UqZMr6x89elTN4+Li1Lxdu3auXBcQEenTp4+ar127tokncdf999+v5p9++qmaFxUVBXGa0OOOGQAAgCEoZgAAAIagmAEAABiCYgYAAGAIihkAAIAhms2pTLtTlpMmTXK0jt0zNN955x3HMwGh8oc//EHNf/3rX6t5dXW1mj/00ENqvmfPHjUvLy8PPNx16Nq1q5oXFhaq+Y033qjmdqfESktL1TwyMlLNs7Oz1dzuWY5237fFixereVpampojPD3zzDNqnpSU5Mr6c+bMUfN///vfav7UU0+p+dChQ9Xc7jT44MGD1Xz27NlqzqlMAAAANAmKGQAAgCEoZgAAAIagmAEAABiCYgYAAGCIsDuV2aZNGzWfO3euo3XsTnE99thjjmcCTHP33XereW1trZr//ve/V3O7feKWhIQENZ8xY4aa250SvXz5sprbnUK1e0afU3anX52ye8ao3+93ZX2Eht0pxZEjR6r5iBEjXLmu3b5dtWqVmldVVal5QUGBmq9cuVLNp02b1oDp/o/ds0HDHXfMAAAADEExAwAAMATFDAAAwBAUMwAAAENQzAAAAAwRdqcyW7TQu2a7du0crdOjRw81P3/+vNORAONkZGSo+aVLl9T88OHDjtaPiYlR83Hjxqm5z+dT88mTJ6v5wIED1XzTpk1qbnfazK3Tl8F27NixUI+AIIiOjlZzu1O4duxOTb711ltqbvdMTLt1unXrpuZ2py9vueUWNXdq3bp1rqzjNdwxAwAAMATFDAAAwBAUMwAAAENQzAAAAAxBMQMAADBE2J3KjIqKUvOUlBRH6zg9xQmEg8jISDUvKipSc7tnQdqdXl67dq2ab9myRc3z8vLU/OTJk2q+f/9+NQfCWatWrdS8vLxczZ944gk1nzRpkprHxcWpeffu3QMP9z/snllrd5ra7vedcMcdMwAAAENQzAAAAAxBMQMAADAExQwAAMAQFDMAAABD+CzLsgK9qKqqSjp06NAU81y3+Ph4NT916pSjdey+LRcuXFDz5cuXq3lOTo6j66JxKisrpX379qEew5FQ7quPPvpIzYcMGeJonZqaGjW3e2Zty5b6QfD77rtPzTdu3OhoHriLfRVcds+UPXPmTBNP0jQefPBBNW9uz8QMtK+4YwYAAGAIihkAAIAhKGYAAACGoJgBAAAYgmIGAABgiLB7Vubp06fVPCEhQc23b9+u5jfddJOav/DCC2q+bNmyBkwHmCEtLU3N77zzTjVv27atmldUVKj5J5984mieK1euOHo9EA7sTjV/9tlnaj5gwIBgjuNYSUmJmi9cuFDNOWXdMNwxAwAAMATFDAAAwBAUMwAAAENQzAAAAAxBMQMAADBE2D0rE80Tz/QD3Me+Co2MjAw1HzNmjJpPnz7d0fplZWVqbndq8vPPP1fzf/7zn2peXV3taJ7mhmdlAgAAeATFDAAAwBAUMwAAAENQzAAAAAxBMQMAADAEpzIRFjg9BriPfQW4j1OZAAAAHkExAwAAMATFDAAAwBAUMwAAAENQzAAAAAxBMQMAADAExQwAAMAQFDMAAABDUMwAAAAMQTEDAAAwBMUMAADAEBQzAAAAQ1DMAAAADEExAwAAMATFDAAAwBAUMwAAAEM0qJhZlhXsOYDr4sXPqBdnRvPixc+oF2dG8xLoM9qgYub3+10ZBggWL35GvTgzmhcvfka9ODOal0CfUZ/VgD9e1NXVyYkTJyQmJkZ8Pp9rwwHXy7Is8fv90qVLF2nRwlt/M8++gqnYV4D7GrqvGlTMAAAAEHze+qMQAABAGKOYAQAAGIJiBgAAYAiKGQAAgCEoZgAAAIagmAEAABiCYgYAAGCI/wLxKCasZ18WwwAAAABJRU5ErkJggg==\n"
          },
          "metadata": {},
          "execution_count": 6
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 3 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmYAAADRCAYAAABmfH5uAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAWpUlEQVR4nO3df1BVdf7H8fcVFUFRAkTzRyhrppuaLYw/GiksTR1ZR3eVrdWwsnLKWnMwtJkMW7dtTN12EEdXM3PTnFxGXHOtZiydkVK0nOiXWqCsmuVKKlx/AcrZP5r4rl/fp8vBc7mfc3k+Zvqjl/d+zhvmfpyXRz4en2VZlgAAACDkWoR6AAAAAPyIYgYAAGAIihkAAIAhKGYAAACGoJgBAAAYgmIGAABgCIoZAACAIShmAAAAhqCYAQAAGIJidp3Ky8vF5/PJ4sWLXVtz586d4vP5ZOfOna6tCXgJ+wpwH/vKG5plMXv99dfF5/PJxx9/HOpRguLQoUMya9YsueOOO6RNmzbi8/mkvLw81GMhzIX7vtq0aZP87ne/k+TkZImOjpZbbrlFsrOz5ezZs6EeDWEs3PeViMj27dtl+PDhkpCQILGxsTJo0CB54403Qj1WyDTLYhbudu/eLXl5eeL3+6Vv376hHgcIC4899pgcOHBApkyZInl5eTJ69GjJz8+XoUOHysWLF0M9HuBJW7ZskXvvvVdqampk/vz58uKLL0pUVJRkZWXJK6+8EurxQqJlqAeA+8aNGydnz56VmJgYWbx4sXz66aehHgnwvIKCAklPT78qS0lJkalTp8r69evlkUceCc1ggIfl5+fLjTfeKB988IFERkaKiMj06dOlT58+8vrrr8usWbNCPGHT446ZjZqaGnn++eclJSVFOnToIG3btpW0tDTZsWOH7XteeeUVSUpKkqioKLnrrrvkiy++uOY1Bw8elIkTJ0pcXJy0adNGUlNTZcuWLQHnuXDhghw8eFAqKioCvjYuLk5iYmICvg5oal7eV/+/lImITJgwQUREDhw4EPD9QLB4eV9VVVXJDTfcUF/KRERatmwpCQkJEhUVFfD94YhiZqOqqkpeffVVSU9Pl4ULF8r8+fPl1KlTMmrUKPUO1N///nfJy8uTGTNmyLPPPitffPGF3H333XLy5Mn613z55ZcyZMgQOXDggMydO1eWLFkibdu2lfHjx0thYeHPzrN3717p27ev5Ofnu/2lAk0m3PbV999/LyIiCQkJjXo/4AYv76v09HT58ssvZd68eVJaWiplZWWyYMEC+fjjjyUnJ8fx9yIsWM3QmjVrLBGx9u3bZ/uay5cvW9XV1VdlZ86csTp16mQ9/PDD9dmRI0csEbGioqKs48eP1+fFxcWWiFizZs2qz+655x6rf//+1qVLl+qzuro664477rBuvvnm+mzHjh2WiFg7duy4JsvNzXX0tS5atMgSEevIkSOO3gc41Zz21U+mTZtmRUREWF9//XWj3g8EEu776ty5c1ZmZqbl8/ksEbFExIqOjrY2b94c8L3hijtmNiIiIqR169YiIlJXVyenT5+Wy5cvS2pqquzfv/+a148fP166du1a//+DBg2SwYMHy7Zt20RE5PTp0/LBBx9IZmam+P1+qaiokIqKCvnhhx9k1KhR8s0338i3335rO096erpYliXz58939wsFmlA47as333xTVq9eLdnZ2XLzzTc7fj/gFi/vq8jISOndu7dMnDhRNmzYIOvWrZPU1FSZMmWK7Nmzx+F3Ijzww/8/Y+3atbJkyRI5ePCg1NbW1uc9e/a85rXab8y9e/eWjRs3iohIaWmpWJYl8+bNk3nz5qnX+89//nPVZgHCUTjsq127dsm0adNk1KhR8uKLL7q6NtAYXt1XTz75pOzZs0f2798vLVr8eK8oMzNTbr31Vpk5c6YUFxdf9zW8hmJmY926dfLggw/K+PHj5ZlnnpHExESJiIiQl156ScrKyhyvV1dXJyIis2fPllGjRqmv6dWr13XNDJguHPZVSUmJjBs3Tvr16ycFBQXSsiW/jSK0vLqvampqZPXq1ZKTk1NfykREWrVqJWPGjJH8/HypqampvxvYXPA7io2CggJJTk6WTZs2ic/nq89zc3PV13/zzTfXZF9//bX06NFDRESSk5NF5McP3IgRI9wfGPAAr++rsrIyGT16tCQmJsq2bdukXbt2Qb8mEIhX99UPP/wgly9flitXrlzza7W1tVJXV6f+WrjjZ8xsREREiIiIZVn1WXFxsezevVt9/ebNm6/6O/e9e/dKcXGxjBkzRkREEhMTJT09Xf72t7/Jd999d837T5069bPzODl+DJjKy/vq+++/l3vvvVdatGgh7733nnTs2DHge4Cm4NV9lZiYKLGxsVJYWCg1NTX1+blz5+Ttt9+WPn36NMt/MqNZ3zF77bXX5N13370mnzlzpmRkZMimTZtkwoQJMnbsWDly5IisWLFCfvnLX8q5c+eueU+vXr1k2LBh8vjjj0t1dbX89a9/lfj4+KuO+y5btkyGDRsm/fv3l0cffVSSk5Pl5MmTsnv3bjl+/LiUlJTYzrp3714ZPny45ObmBvyBysrKSlm6dKmIiHz44Yci8uM/4hcbGyuxsbHy5JNPNuTbAzRKuO6r0aNHy+HDhyUnJ0eKioqkqKio/tc6deokI0eObMB3B2iccNxXERERMnv2bHnuuedkyJAhkpWVJVeuXJHVq1fL8ePHZd26dc6+SeEiZOdBQ+in48d2/x07dsyqq6uz/vznP1tJSUlWZGSkdfvtt1tbt261pk6daiUlJdWv9dPx40WLFllLliyxunfvbkVGRlppaWlWSUnJNdcuKyuzsrKyrM6dO1utWrWyunbtamVkZFgFBQX1r7ne48c/zaT997+zA24K9331c1/bXXfddR3fOcBeuO8ry7Ks9evXW4MGDbJiY2OtqKgoa/DgwVddo7nxWdb/3PsEAABAyPAzZgAAAIagmAEAABiCYgYAAGAIihkAAIAhKGYAAACGoJgBAAAYokH/wGxdXZ2cOHFCYmJirnrcAxBqlmWJ3++XLl26XPWsNS9gX8FU7CvAfQ3dVw0qZidOnJDu3bu7NhzgtmPHjkm3bt1CPYYj7CuYjn0FuC/QvmrQH4ViYmJcGwgIBi9+Rr04M5oXL35GvTgzmpdAn9EGFTNuB8N0XvyMenFmNC9e/Ix6cWY0L4E+o9764QEAAIAwRjEDAAAwBMUMAADAEBQzAAAAQ1DMAAAADEExAwAAMATFDAAAwBAUMwAAAENQzAAAAAxBMQMAADAExQwAAMAQFDMAAABDUMwAAAAMQTEDAAAwBMUMAADAEBQzAAAAQ1DMAAAADEExAwAAMATFDAAAwBAUMwAAAEO0DPUAaFoDBw5U8wULFqh5RkaGmsfHx6v56dOnGzUXvCk2NlbNH3nkETVfunSpmnfs2FHNc3NzHa3vVGlpqZrfc889av7dd9+peW1trSvzAAB3zAAAAAxBMQMAADAExQwAAMAQFDMAAABDUMwAAAAMwanMMNWqVSs1z87OVvOxY8eqeU1NjZpbltW4weBJPp9PzRctWqTm06ZNU/OXX37ZlXnsPn9XrlxR84sXL6p5ly5d1NzutGZhYaGa33///WpeV1en5sDPiYyMVPPWrVureadOndR86tSpjq5rd9o5MTHR0Tp2tm/frua/+c1v1Pz8+fOuXNdruGMGAABgCIoZAACAIShmAAAAhqCYAQAAGIJiBgAAYAhOZXqc3SmdVatWqfnkyZPVvLq6Ws0zMzPV/MyZMw2YDuHivvvuU3O705du8fv9am73bMqFCxequd3p0R49eqh5v3791Nzu2Z12r//ss8/UHBARSUpKUnO7078DBgwI5ji23DqFP2LECDW3ewbthAkT1Pz99993ZR5TcccMAADAEBQzAAAAQ1DMAAAADEExAwAAMATFDAAAwBCcyvQ4u9NyDzzwgKN1VqxYoeZvv/2245ngXW3atFHzuXPnOlrn6NGjav7RRx+p+a5du9T8nXfeUfPy8nJH89ixW8cub9eunZo/99xzam73DE27Z3qieTl8+LCaN7dnEUdHR6v5ypUr1fy2225T83Pnzrk2UyhxxwwAAMAQFDMAAABDUMwAAAAMQTEDAAAwBMUMAADAEJzK9Ihhw4ap+ZIlSxytc/78eTVfvHix45kQfuxOC+bl5Tlap6ioSM0PHTrkeCYvmDhxoprPnj1bze1OrQKNcfHiRTV3ut/sTk1nZGSo+U033eRofac6d+6s5kOGDFHz7du3q3nv3r3VPCUlRc3Hjx+v5u+++66ar1mzRs0biztmAAAAhqCYAQAAGIJiBgAAYAiKGQAAgCEoZgAAAIbgVKZh2rZtq+b5+flqHh8fr+Z+v1/NJ0+erObffvttA6ZDuKutrVXz1atXN/Ek4SEuLk7NOZUJNy1YsEDNX375ZVfWt1unX79+ap6ZmanmU6dOdXTdLVu2qHlpaama2z2b1u5U+Q033OBontTUVDXfsGGDml+6dMnR+j/hjhkAAIAhKGYAAACGoJgBAAAYgmIGAABgCIoZAACAITiVGSJ2py+XL1+u5gMGDFDzqqoqNc/KylLzrVu3NmA6AG6YMWOGmj/66KNNPAnC2dChQ4O6/vHjxx3lv/3tb1257qRJk9Tc7uvt1q2bK9e1k5SUpOZjxoxR88LCwkZdhztmAAAAhqCYAQAAGIJiBgAAYAiKGQAAgCEoZgAAAIbgVGaIZGRkqPmUKVMcrbN+/Xo1t3vGGIDG69y5s6PXb9y4MUiTIBy89tprav7QQw85WmfEiBFqPmfOHDX/8MMP1byoqMjRde2eTWn3TEy7Z/GWlJSoeUpKipoH+/SlU//4xz/UvGXLxlUs7pgBAAAYgmIGAABgCIoZAACAIShmAAAAhqCYAQAAGMJnWZYV6EVVVVXSoUOHppgn7Lzwwgtq/sQTT6h5fHy8mm/evFnN7U7vVFZWBh4ujFRWVkr79u1DPYYj7CtzxcXFqfm+ffvUvGfPnmreo0cPNT969Gij5mpq7Kvg6t69u5ofOXIkqNe9cOGCmts9w/Wtt95S888//1zN+/btq+Y+n0/NG1BDrsvSpUvVfNmyZWo+ffp0NY+NjXV0XbvvZ6B9xR0zAAAAQ1DMAAAADEExAwAAMATFDAAAwBAUMwAAAENwKtMlds8q27Bhg5rbnb48c+aMmqelpan5V1991YDpwh+nx+Cmp59+Ws3/8pe/qPnJkyfVfODAgY5ebxr2VXBFRkaqud0pwocffjiY48jFixfVvKqqSs07duyo5i1a6Pd8gn0qs7i4WM2HDx+u5jU1Na5c1ylOZQIAAHgExQwAAMAQFDMAAABDUMwAAAAMQTEDAAAwRMtQD+A1ycnJau7W6cusrCw15/Ql4D67U2UzZsxwtM6KFSvU3CunLxEa1dXVap6dna3mqampaj5gwABX5omKinKUh8ry5cvVPCcnR81DdfqysbhjBgAAYAiKGQAAgCEoZgAAAIagmAEAABiCYgYAAGAITmXasHvG3Zw5c9Tc7vTlvn371PyPf/yjmv/rX/8KPBzQxFq3bq3mEyZMUPNZs2Y5Wn/Xrl1q3qtXLzV/9dVX1XzKlClq3rNnTzWPjY1V81/84hdqbufEiRNqPnbsWEfrOFVaWqrmhw4dCup1EVx+v1/Nf/WrX6m53SnivLw812Zyg90zNCsqKtR827Ztav7UU0+5NpOJuGMGAABgCIoZAACAIShmAAAAhqCYAQAAGIJiBgAAYAifZVlWoBdVVVVJhw4dmmKeJhcdHa3mdqdB7rzzTjU/e/asmo8cOVLNP/nkk8DDocEqKyulffv2oR7DERP3ld2przVr1qh5//79gzkOAvjTn/6k5s8//7wr67OvvGHYsGFqvnPnzqYdJACfz6fmM2fOVPP8/PxgjhMygfYVd8wAAAAMQTEDAAAwBMUMAADAEBQzAAAAQ1DMAAAADNFsnpUZFRWl5nanzexOX1ZWVqr55MmT1ZzTlzBRRESEmufm5qq509OX58+fV/OtW7c6Wsep9PR0Ne/UqZMr6x89elTN4+Li1Lxdu3auXBcQEenTp4+ar127tokncdf999+v5p9++qmaFxUVBXGa0OOOGQAAgCEoZgAAAIagmAEAABiCYgYAAGAIihkAAIAhms2pTLtTlpMmTXK0jt0zNN955x3HMwGh8oc//EHNf/3rX6t5dXW1mj/00ENqvmfPHjUvLy8PPNx16Nq1q5oXFhaq+Y033qjmdqfESktL1TwyMlLNs7Oz1dzuWY5237fFixereVpampojPD3zzDNqnpSU5Mr6c+bMUfN///vfav7UU0+p+dChQ9Xc7jT44MGD1Xz27NlqzqlMAAAANAmKGQAAgCEoZgAAAIagmAEAABiCYgYAAGCIsDuV2aZNGzWfO3euo3XsTnE99thjjmcCTHP33XereW1trZr//ve/V3O7feKWhIQENZ8xY4aa250SvXz5sprbnUK1e0afU3anX52ye8ao3+93ZX2Eht0pxZEjR6r5iBEjXLmu3b5dtWqVmldVVal5QUGBmq9cuVLNp02b1oDp/o/ds0HDHXfMAAAADEExAwAAMATFDAAAwBAUMwAAAENQzAAAAAwRdqcyW7TQu2a7du0crdOjRw81P3/+vNORAONkZGSo+aVLl9T88OHDjtaPiYlR83Hjxqm5z+dT88mTJ6v5wIED1XzTpk1qbnfazK3Tl8F27NixUI+AIIiOjlZzu1O4duxOTb711ltqbvdMTLt1unXrpuZ2py9vueUWNXdq3bp1rqzjNdwxAwAAMATFDAAAwBAUMwAAAENQzAAAAAxBMQMAADBE2J3KjIqKUvOUlBRH6zg9xQmEg8jISDUvKipSc7tnQdqdXl67dq2ab9myRc3z8vLU/OTJk2q+f/9+NQfCWatWrdS8vLxczZ944gk1nzRpkprHxcWpeffu3QMP9z/snllrd5ra7vedcMcdMwAAAENQzAAAAAxBMQMAADAExQwAAMAQFDMAAABD+CzLsgK9qKqqSjp06NAU81y3+Ph4NT916pSjdey+LRcuXFDz5cuXq3lOTo6j66JxKisrpX379qEew5FQ7quPPvpIzYcMGeJonZqaGjW3e2Zty5b6QfD77rtPzTdu3OhoHriLfRVcds+UPXPmTBNP0jQefPBBNW9uz8QMtK+4YwYAAGAIihkAAIAhKGYAAACGoJgBAAAYgmIGAABgiLB7Vubp06fVPCEhQc23b9+u5jfddJOav/DCC2q+bNmyBkwHmCEtLU3N77zzTjVv27atmldUVKj5J5984mieK1euOHo9EA7sTjV/9tlnaj5gwIBgjuNYSUmJmi9cuFDNOWXdMNwxAwAAMATFDAAAwBAUMwAAAENQzAAAAAxBMQMAADBE2D0rE80Tz/QD3Me+Co2MjAw1HzNmjJpPnz7d0fplZWVqbndq8vPPP1fzf/7zn2peXV3taJ7mhmdlAgAAeATFDAAAwBAUMwAAAENQzAAAAAxBMQMAADAEpzIRFjg9BriPfQW4j1OZAAAAHkExAwAAMATFDAAAwBAUMwAAAENQzAAAAAxBMQMAADAExQwAAMAQFDMAAABDUMwAAAAMQTEDAAAwBMUMAADAEBQzAAAAQ1DMAAAADEExAwAAMATFDAAAwBAUMwAAAEM0qJhZlhXsOYDr4sXPqBdnRvPixc+oF2dG8xLoM9qgYub3+10ZBggWL35GvTgzmhcvfka9ODOal0CfUZ/VgD9e1NXVyYkTJyQmJkZ8Pp9rwwHXy7Is8fv90qVLF2nRwlt/M8++gqnYV4D7GrqvGlTMAAAAEHze+qMQAABAGKOYAQAAGIJiBgAAYAiKGQAAgCEoZgAAAIagmAEAABiCYgYAAGCI/wLxKCasZ18WwwAAAABJRU5ErkJggg==\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Now we are ready to build out network!  \n",
        "For simplicity, we will build a simple CNN network to compare different optimizer"
      ],
      "metadata": {
        "id": "bjmetuc4W_nx"
      }
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "41zAlR231C4S"
      },
      "source": [
        "### Create a CNN Model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "lPrV8Y4s1C4T"
      },
      "outputs": [],
      "source": [
        "class Simple_CNN(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(Simple_CNN, self).__init__()\n",
        "\n",
        "        self.conv1 = nn.Conv2d(1, 32, kernel_size=5, padding=2)\n",
        "        self.pool1 = nn.MaxPool2d(kernel_size=3, stride=3)\n",
        "        self.conv2 = nn.Conv2d(32, 64, kernel_size=5)\n",
        "        self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)\n",
        "        self.flatten = nn.Flatten()\n",
        "\n",
        "        self.fc = nn.Linear(64 * 2 * 2, 10)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.pool1(torch.relu(self.conv1(x)))\n",
        "        x = self.pool2(torch.relu(self.conv2(x)))\n",
        "\n",
        "        x = self.flatten(x)\n",
        "        x = self.fc(x)\n",
        "\n",
        "        return F.log_softmax(x, dim=1)\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Print summary table of the model"
      ],
      "metadata": {
        "id": "vKgjEDzJTVKs"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = Simple_CNN()\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "model.to(device)\n",
        "\n",
        "summary(model, (1, 28, 28))\n"
      ],
      "metadata": {
        "id": "Am6E7w0sQf93",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "277eec33-b846-4ee5-ef89-d52d420d469e"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Conv2d-1           [-1, 32, 28, 28]             832\n",
            "         MaxPool2d-2             [-1, 32, 9, 9]               0\n",
            "            Conv2d-3             [-1, 64, 5, 5]          51,264\n",
            "         MaxPool2d-4             [-1, 64, 2, 2]               0\n",
            "           Flatten-5                  [-1, 256]               0\n",
            "            Linear-6                   [-1, 10]           2,570\n",
            "================================================================\n",
            "Total params: 54,666\n",
            "Trainable params: 54,666\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.00\n",
            "Forward/backward pass size (MB): 0.23\n",
            "Params size (MB): 0.21\n",
            "Estimated Total Size (MB): 0.44\n",
            "----------------------------------------------------------------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I4EPM2hf1C4U"
      },
      "source": [
        "### Fitting model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "Io5aFLpw1C4U"
      },
      "outputs": [],
      "source": [
        "def fit(model, optimizer, train_loader):\n",
        "    # Use GPU to train\n",
        "    model = model.to(device)\n",
        "\n",
        "    EPOCHS = 20\n",
        "    model.train()\n",
        "    loss_values = []\n",
        "\n",
        "    for epoch in range(EPOCHS):\n",
        "        correct = 0\n",
        "        running_loss = 0.0\n",
        "        total_samples = 0\n",
        "\n",
        "        for batch_idx, (X_batch, y_batch) in enumerate(train_loader):\n",
        "            X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n",
        "\n",
        "            optimizer.zero_grad()\n",
        "            output = model(X_batch)\n",
        "\n",
        "            loss = F.nll_loss(output, y_batch)\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "            running_loss += loss.item()\n",
        "\n",
        "        epoch_loss = running_loss / len(train_loader)\n",
        "        loss_values.append(epoch_loss)\n",
        "\n",
        "        print('[Epoch %d] Loss: %.3f' % (epoch + 1, epoch_loss))\n",
        "\n",
        "    return loss_values\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "First, we will fit the Adam Optimizer with learning rate of 5e-4 and beta as (0.9,0.999) as the original paper suggests"
      ],
      "metadata": {
        "id": "m8NJMhtvTqwG"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Nj3lLjLY1C4U",
        "outputId": "c559a4ef-0958-4a6f-df3b-f50f82f7ae14"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Epoch 1] Loss: 0.217\n",
            "[Epoch 2] Loss: 0.061\n",
            "[Epoch 3] Loss: 0.044\n",
            "[Epoch 4] Loss: 0.034\n",
            "[Epoch 5] Loss: 0.028\n",
            "[Epoch 6] Loss: 0.023\n",
            "[Epoch 7] Loss: 0.018\n",
            "[Epoch 8] Loss: 0.016\n",
            "[Epoch 9] Loss: 0.014\n",
            "[Epoch 10] Loss: 0.011\n",
            "[Epoch 11] Loss: 0.010\n",
            "[Epoch 12] Loss: 0.008\n",
            "[Epoch 13] Loss: 0.007\n",
            "[Epoch 14] Loss: 0.007\n",
            "[Epoch 15] Loss: 0.007\n",
            "[Epoch 16] Loss: 0.005\n",
            "[Epoch 17] Loss: 0.005\n",
            "[Epoch 18] Loss: 0.004\n",
            "[Epoch 19] Loss: 0.005\n",
            "[Epoch 20] Loss: 0.002\n"
          ]
        }
      ],
      "source": [
        "cnn_adam = Simple_CNN().to(device)\n",
        "Adam = torch.optim.Adam(cnn_adam.parameters(), lr=5e-4, betas=(0.9, 0.999))\n",
        "Adam_model = fit(cnn_adam, Adam, train_loader)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hTWf4dn-f8Ey"
      },
      "source": [
        "We also train with a naive SGD Optimizer with learing rate of 5e-4"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ERS3cCIwf7X5",
        "outputId": "dd6143b5-2be0-4cfd-dae6-1cde092570e1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Epoch 1] Loss: 2.033\n",
            "[Epoch 2] Loss: 1.323\n",
            "[Epoch 3] Loss: 0.742\n",
            "[Epoch 4] Loss: 0.510\n",
            "[Epoch 5] Loss: 0.407\n",
            "[Epoch 6] Loss: 0.348\n",
            "[Epoch 7] Loss: 0.309\n",
            "[Epoch 8] Loss: 0.280\n",
            "[Epoch 9] Loss: 0.258\n",
            "[Epoch 10] Loss: 0.239\n",
            "[Epoch 11] Loss: 0.224\n",
            "[Epoch 12] Loss: 0.211\n",
            "[Epoch 13] Loss: 0.200\n",
            "[Epoch 14] Loss: 0.191\n",
            "[Epoch 15] Loss: 0.182\n",
            "[Epoch 16] Loss: 0.174\n",
            "[Epoch 17] Loss: 0.167\n",
            "[Epoch 18] Loss: 0.161\n",
            "[Epoch 19] Loss: 0.155\n",
            "[Epoch 20] Loss: 0.150\n"
          ]
        }
      ],
      "source": [
        "cnn_sgd = Simple_CNN().to(device)\n",
        "SGD_optimizer = torch.optim.SGD(cnn_sgd.parameters(), lr=5e-4)\n",
        "SGD_model = fit(cnn_sgd, SGD_optimizer, train_loader)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Adagrad with learning rate of 5e-4"
      ],
      "metadata": {
        "id": "dbzBGDJqUH8o"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "cnn_Adagrad = Simple_CNN().to(device)\n",
        "Adagrad = torch.optim.Adagrad(cnn_Adagrad.parameters(), lr=5e-4)\n",
        "Adagrad_model = fit(cnn_Adagrad, Adagrad, train_loader)"
      ],
      "metadata": {
        "id": "j4gSHqRiSw8t",
        "outputId": "79276cc8-9669-40fd-df3c-2620381af72d",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Epoch 1] Loss: 0.793\n",
            "[Epoch 2] Loss: 0.358\n",
            "[Epoch 3] Loss: 0.279\n",
            "[Epoch 4] Loss: 0.240\n",
            "[Epoch 5] Loss: 0.216\n",
            "[Epoch 6] Loss: 0.199\n",
            "[Epoch 7] Loss: 0.186\n",
            "[Epoch 8] Loss: 0.176\n",
            "[Epoch 9] Loss: 0.167\n",
            "[Epoch 10] Loss: 0.160\n",
            "[Epoch 11] Loss: 0.154\n",
            "[Epoch 12] Loss: 0.148\n",
            "[Epoch 13] Loss: 0.144\n",
            "[Epoch 14] Loss: 0.139\n",
            "[Epoch 15] Loss: 0.135\n",
            "[Epoch 16] Loss: 0.132\n",
            "[Epoch 17] Loss: 0.129\n",
            "[Epoch 18] Loss: 0.126\n",
            "[Epoch 19] Loss: 0.123\n",
            "[Epoch 20] Loss: 0.121\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "RMSprop with learning rate of 5e-4"
      ],
      "metadata": {
        "id": "P7VYtWJ3UKzq"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "cnn_RMSprop = Simple_CNN().to(device)\n",
        "RMSprop = torch.optim.RMSprop(cnn_RMSprop.parameters(), lr=5e-4)\n",
        "RMSprop_model = fit(cnn_RMSprop, RMSprop, train_loader)"
      ],
      "metadata": {
        "id": "ElLmS0TaSxEb",
        "outputId": "11d2b9dd-4483-4c08-9ba8-d9025dc82149",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Epoch 1] Loss: 0.209\n",
            "[Epoch 2] Loss: 0.070\n",
            "[Epoch 3] Loss: 0.051\n",
            "[Epoch 4] Loss: 0.040\n",
            "[Epoch 5] Loss: 0.033\n",
            "[Epoch 6] Loss: 0.028\n",
            "[Epoch 7] Loss: 0.024\n",
            "[Epoch 8] Loss: 0.021\n",
            "[Epoch 9] Loss: 0.018\n",
            "[Epoch 10] Loss: 0.016\n",
            "[Epoch 11] Loss: 0.013\n",
            "[Epoch 12] Loss: 0.011\n",
            "[Epoch 13] Loss: 0.010\n",
            "[Epoch 14] Loss: 0.009\n",
            "[Epoch 15] Loss: 0.008\n",
            "[Epoch 16] Loss: 0.007\n",
            "[Epoch 17] Loss: 0.006\n",
            "[Epoch 18] Loss: 0.006\n",
            "[Epoch 19] Loss: 0.004\n",
            "[Epoch 20] Loss: 0.005\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "L-BFGS with learning rate of 5e-4, max_iteration of 1 and history size = 20.\n",
        "\n",
        "\n",
        "Although L-BFGS holds theoretical superiority by leveraging second-order derivatives, the practical limitation of computational resources constrains the increase useage of both the maximum iteration count and history size to attain its optimal efficacy. It is noteworthy that the study conducted by Le et al., titled \"On Optimization Methods for Deep Learning,\" attests to the achievement of state-of-the-art outcomes in convolutional neural networks through the application of L-BFGS."
      ],
      "metadata": {
        "id": "AMIx_rnZVMPH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "cnn_LBFGS = Simple_CNN()\n",
        "cnn_LBFGS.to(device)\n",
        "LBFGS = torch.optim.LBFGS(cnn_LBFGS.parameters(),lr=5e-4,max_iter=1,history_size=20,\n",
        "                          line_search_fn='strong_wolfe')\n",
        "\n",
        "epochs = 20\n",
        "LBFGS_loss = []\n",
        "\n",
        "for epoch in range(epochs):\n",
        "    running_loss = 0.0\n",
        "\n",
        "    for batch_idx, (X_batch, y_batch) in enumerate(train_loader):\n",
        "      X_batch, y_batch = X_batch.to(device), y_batch.to(device)\n",
        "\n",
        "      def closure():\n",
        "        # Zero gradients\n",
        "        LBFGS.zero_grad()\n",
        "        y_pred = cnn_LBFGS(X_batch)\n",
        "        loss = F.nll_loss(y_pred, y_batch)\n",
        "        loss.backward()\n",
        "        return loss\n",
        "\n",
        "      LBFGS.step(closure)\n",
        "      loss = closure()\n",
        "      running_loss += loss.item()\n",
        "\n",
        "    epoch_loss = running_loss / len(train_loader)\n",
        "    LBFGS_loss.append(epoch_loss)\n",
        "\n",
        "    print('[Epoch %d] Loss: %.3f' % (epoch + 1, epoch_loss))\n",
        "\n",
        ""
      ],
      "metadata": {
        "id": "hyUKS2mkSxHC",
        "outputId": "37621e5c-eccb-4367-9b02-e2c8d1acc90a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Epoch 1] Loss: 0.268\n",
            "[Epoch 2] Loss: 0.155\n",
            "[Epoch 3] Loss: 0.134\n",
            "[Epoch 4] Loss: 0.117\n",
            "[Epoch 5] Loss: 0.105\n",
            "[Epoch 6] Loss: 0.097\n",
            "[Epoch 7] Loss: 0.091\n",
            "[Epoch 8] Loss: 0.084\n",
            "[Epoch 9] Loss: 0.079\n",
            "[Epoch 10] Loss: 0.075\n",
            "[Epoch 11] Loss: 0.069\n",
            "[Epoch 12] Loss: 0.066\n",
            "[Epoch 13] Loss: 0.062\n",
            "[Epoch 14] Loss: 0.061\n",
            "[Epoch 15] Loss: 0.058\n",
            "[Epoch 16] Loss: 0.056\n",
            "[Epoch 17] Loss: 0.052\n",
            "[Epoch 18] Loss: 0.051\n",
            "[Epoch 19] Loss: 0.048\n",
            "[Epoch 20] Loss: 0.048\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RWJTwsi4hprD"
      },
      "source": [
        "Plot Training Loss for comparision"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 472
        },
        "id": "niVdUHhp1C4U",
        "outputId": "bfc626e1-3563-4202-801d-22fb46b53b7b"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkAAAAHHCAYAAABXx+fLAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACY5UlEQVR4nOzdd3wUdf7H8ddsT+8VQggJUqVDaAoiEooonj/FShF7FytWuPPwxIYKinoocp4KnoodaSLSaxCQTkIoKbT0ZDe7O78/NplkSYAEEjbl83w89rE7M9+Z/ewGyJvvfOc7iqqqKkIIIYQQTYjO0wUIIYQQQlxsEoCEEEII0eRIABJCCCFEkyMBSAghhBBNjgQgIYQQQjQ5EoCEEEII0eRIABJCCCFEkyMBSAghhBBNjgQgIYQQQjQ5EoCEqMfGjRtHy5Ytz2vfyZMnoyhK7RYkRBXmzJmDoihs3LjR06UIUW0SgIQ4D4qiVOuxfPlyT5fqEePGjcPX19fTZTQaZQHjTI+1a9d6ukQhGhyDpwsQoiH6z3/+47Y8d+5cFi9eXGl9u3btLuh9PvroI5xO53nt+/zzz/PMM89c0PuL+uXvf/87cXFxldYnJCR4oBohGjYJQEKch9tuu81tee3atSxevLjS+tMVFhbi7e1d7fcxGo3nVR+AwWDAYJC/4g1FQUEBPj4+Z20zbNgwevTocZEqEqJxk1NgQtSRgQMH0rFjRzZt2sTll1+Ot7c3zz77LADfffcdI0aMIDo6GrPZTHx8PP/4xz9wOBxuxzh9DFBqaiqKovD666/z4YcfEh8fj9lspmfPnmzYsMFt36rGACmKwoMPPsiCBQvo2LEjZrOZDh06sHDhwkr1L1++nB49emCxWIiPj+eDDz6o9XFFX331Fd27d8fLy4vQ0FBuu+02jhw54tYmIyOD8ePH07x5c8xmM1FRUVx77bWkpqZqbTZu3EhSUhKhoaF4eXkRFxfHHXfcUa0a3nvvPTp06IDZbCY6OpoHHniA7OxsbfuDDz6Ir68vhYWFlfa9+eabiYyMdPu5/fLLL1x22WX4+Pjg5+fHiBEj2LFjh9t+ZacI9+/fz/Dhw/Hz8+PWW2+tVr1nU/HPx1tvvUVsbCxeXl4MGDCA7du3V2q/bNkyrdbAwECuvfZadu7cWandkSNHmDBhgvbnNS4ujvvuuw+bzebWzmq1MnHiRMLCwvDx8eG6667j2LFjlY5Xne+oOj93IS6E/PdQiDp04sQJhg0bxk033cRtt91GREQE4BrT4evry8SJE/H19WXZsmW8+OKL5Obm8tprr53zuJ9//jl5eXncc889KIrCtGnT+Nvf/saBAwfO2Wu0cuVKvvnmG+6//378/Px45513uP7660lLSyMkJASALVu2MHToUKKiopgyZQoOh4O///3vhIWFXfiXUmrOnDmMHz+enj178sorr5CZmcnbb7/NqlWr2LJlC4GBgQBcf/317Nixg4ceeoiWLVuSlZXF4sWLSUtL05aHDBlCWFgYzzzzDIGBgaSmpvLNN9+cs4bJkyczZcoUBg8ezH333cfu3bt5//332bBhA6tWrcJoNDJ69GhmzpzJTz/9xA033KDtW1hYyA8//MC4cePQ6/WA69To2LFjSUpK4tVXX6WwsJD333+f/v37s2XLFrcwa7fbSUpKon///rz++uvV6hnMycnh+PHjbusURdF+bmXmzp1LXl4eDzzwAMXFxbz99tsMGjSIbdu2aX8GlyxZwrBhw2jVqhWTJ0+mqKiId999l379+rF582at1qNHj9KrVy+ys7O5++67adu2LUeOHOF///sfhYWFmEwm7X0feughgoKCeOmll0hNTWX69Ok8+OCDzJs3T2tT3e/oXD93IS6YKoS4YA888IB6+l+nAQMGqIA6a9asSu0LCwsrrbvnnntUb29vtbi4WFs3duxYNTY2VltOSUlRATUkJEQ9efKktv67775TAfWHH37Q1r300kuVagJUk8mk7tu3T1u3detWFVDfffddbd3IkSNVb29v9ciRI9q6vXv3qgaDodIxqzJ27FjVx8fnjNttNpsaHh6uduzYUS0qKtLW//jjjyqgvvjii6qqquqpU6dUQH3ttdfOeKxvv/1WBdQNGzacs66KsrKyVJPJpA4ZMkR1OBza+hkzZqiA+vHHH6uqqqpOp1Nt1qyZev3117vtP3/+fBVQV6xYoaqqqubl5amBgYHqXXfd5dYuIyNDDQgIcFs/duxYFVCfeeaZatX6ySefqECVD7PZrLUr+/Ph5eWlHj58WFu/bt06FVAfe+wxbV2XLl3U8PBw9cSJE9q6rVu3qjqdTh0zZoy2bsyYMapOp6vy+3U6nW71DR48WFunqqr62GOPqXq9Xs3Ozq7Rd1Sdn7sQF0pOgQlRh8xmM+PHj6+03svLS3udl5fH8ePHueyyyygsLGTXrl3nPO7o0aMJCgrSli+77DIADhw4cM59Bw8eTHx8vLbcqVMn/P39tX0dDgdLlixh1KhRREdHa+0SEhIYNmzYOY9fHRs3biQrK4v7778fi8WirR8xYgRt27blp59+Alzfk8lkYvny5Zw6darKY5X1FP3444+UlJRUu4YlS5Zgs9l49NFH0enK/ym866678Pf312pQFIUbbriBn3/+mfz8fK3dvHnzaNasGf379wdg8eLFZGdnc/PNN3P8+HHtodfrSUxM5LfffqtUw3333VftegFmzpzJ4sWL3R6//PJLpXajRo2iWbNm2nKvXr1ITEzk559/BiA9PZ3k5GTGjRtHcHCw1q5Tp05cddVVWjun08mCBQsYOXJklWOPTj8devfdd7utu+yyy3A4HBw8eBCo/ndUnZ+7EBdKApAQdahZs2ZupwjK7Nixg+uuu46AgAD8/f0JCwvTBlDn5OSc87gtWrRwWy4LQ9X5ZXH6vmX7l+2blZVFUVFRlVcW1dbVRmW/ENu0aVNpW9u2bbXtZrOZV199lV9++YWIiAguv/xypk2bRkZGhtZ+wIABXH/99UyZMoXQ0FCuvfZaPvnkE6xW63nVYDKZaNWqlbYdXIGzqKiI77//HoD8/Hx+/vlnbrjhBu0X/t69ewEYNGgQYWFhbo9FixaRlZXl9j4Gg4HmzZuf+8uqoFevXgwePNjtccUVV1Rq17p160rrLrnkEm38zNm+/3bt2nH8+HEKCgo4duwYubm5dOzYsVr1nevPZXW/o+r83IW4UDIGSIg6VLGnp0x2djYDBgzA39+fv//978THx2OxWNi8eTNPP/10tS57LxtzcjpVVet0X0949NFHGTlyJAsWLODXX3/lhRde4JVXXmHZsmV07doVRVH43//+x9q1a/nhhx/49ddfueOOO3jjjTdYu3ZtrcxH1Lt3b1q2bMn8+fO55ZZb+OGHHygqKmL06NFam7Kf23/+8x8iIyMrHeP0K/LMZrNbz1NjcK4/WzX5js71cxfiQkkAEuIiW758OSdOnOCbb77h8ssv19anpKR4sKpy4eHhWCwW9u3bV2lbVevOR2xsLAC7d+9m0KBBbtt2796tbS8THx/P448/zuOPP87evXvp0qULb7zxBp999pnWpnfv3vTu3Zt//vOffP7559x66618+eWX3HnnneesoVWrVtp6m81GSkoKgwcPdmt/44038vbbb5Obm8u8efNo2bIlvXv3dqsRXN/f6ftebGU9LRXt2bNHGzxc8bOfbteuXYSGhuLj44OXlxf+/v5VXkF2Pmr6HVXn5y7E+Wpc//0QogEo+19yxR4Xm83Ge++956mS3Oj1egYPHsyCBQs4evSotn7fvn1Vjjc5Hz169CA8PJxZs2a5nar65Zdf2LlzJyNGjABcV1oVFxe77RsfH4+fn5+236lTpyr1XnXp0gXgrKfBBg8ejMlk4p133nHbf/bs2eTk5Gg1lBk9ejRWq5VPP/2UhQsXcuONN7ptT0pKwt/fn6lTp1Y5Fqmqy8HryoIFC9ymE1i/fj3r1q3TxnBFRUXRpUsXPv30U7dL/rdv386iRYsYPnw4ADqdjlGjRvHDDz9UeZuLmvYaVvc7qs7PXYgLJT1AQlxkffv2JSgoiLFjx/Lwww+jKAr/+c9/6tUpqMmTJ7No0SL69evHfffdh8PhYMaMGXTs2JHk5ORqHaOkpISXX3650vrg4GDuv/9+Xn31VcaPH8+AAQO4+eabtcvgW7ZsyWOPPQa4ei2uvPJKbrzxRtq3b4/BYODbb78lMzOTm266CYBPP/2U9957j+uuu474+Hjy8vL46KOP8Pf3136RVyUsLIxJkyYxZcoUhg4dyjXXXMPu3bt577336NmzZ6VJLbt160ZCQgLPPfccVqvV7fQXgL+/P++//z6333473bp146abbiIsLIy0tDR++ukn+vXrx4wZM6r13Z3JL7/8UuUg+b59+7r1YiUkJNC/f3/uu+8+rFYr06dPJyQkhKeeekpr89prrzFs2DD69OnDhAkTtMvgAwICmDx5stZu6tSpLFq0iAEDBnD33XfTrl070tPT+eqrr1i5cqU2CL06qvsdVefnLsQF89wFaEI0Hme6DL5Dhw5Vtl+1apXau3dv1cvLS42Ojlafeuop9ddff1UB9bffftPaneky+KouDwbUl156SVs+02XwDzzwQKV9Y2Nj1bFjx7qtW7p0qdq1a1fVZDKp8fHx6r///W/18ccfVy0Wyxm+hXJll3lX9YiPj9fazZs3T+3atatqNpvV4OBg9dZbb3W7fPv48ePqAw88oLZt21b18fFRAwIC1MTERHX+/Plam82bN6s333yz2qJFC9VsNqvh4eHq1VdfrW7cuPGcdaqq67L3tm3bqkajUY2IiFDvu+8+9dSpU1W2fe6551RATUhIOOPxfvvtNzUpKUkNCAhQLRaLGh8fr44bN86tnnNNE3C6s10GD6iffPKJqqrufz7eeOMNNSYmRjWbzepll12mbt26tdJxlyxZovbr10/18vJS/f391ZEjR6p//fVXpXYHDx5Ux4wZo4aFhalms1lt1aqV+sADD6hWq9WtvtMvlf/tt98q/ZmuzndUnZ+7EBdKUdV69N9OIUS9NmrUKHbs2FHlGBPheampqcTFxfHaa6/xxBNPeLocIeo1GQMkhKhSUVGR2/LevXv5+eefGThwoGcKEkKIWiRjgIQQVWrVqhXjxo3T5sR5//33MZlMbuNIhBCioZIAJISo0tChQ/niiy/IyMjAbDbTp08fpk6dWuUke0II0dDIGCAhhBBCNDkyBkgIIYQQTY4EICGEEEI0OTIGqApOp5OjR4/i5+dX6W7HQgghhKifVFUlLy+P6Ojoc95rTwJQFY4ePUpMTIynyxBCCCHEeTh06BDNmzc/axsJQFXw8/MDXF+gv7+/h6sRQgghRHXk5uYSExOj/R4/GwlAVSg77eXv7y8BSAghhGhgqjN8RQZBCyGEEKLJkQAkhBBCiCZHApAQQgghmhwZAySEEKLJcjgclJSUeLoMUU1GoxG9Xl8rx5IAJIQQoslRVZWMjAyys7M9XYqoocDAQCIjIy94nj4JQEIIIZqcsvATHh6Ot7e3THrbAKiqSmFhIVlZWQBERUVd0PEkAAkhhGhSHA6HFn5CQkI8XY6oAS8vLwCysrIIDw+/oNNhMghaCCFEk1I25sfb29vDlYjzUfZzu9CxWxKAhBBCNEly2qthqq2fmwQgIYQQQjQ5EoCEEEKIRmzy5Ml06dLF02XUOxKAhBBCiAZmzZo16PV6RowY4elSGiwJQBeTqsKpVMg57OlKhBBCNGCzZ8/moYceYsWKFRw9etTT5TRIEoAupkXPw9udYe37nq5ECCFEA5Wfn8+8efO47777GDFiBHPmzHHb/q9//YuIiAj8/PyYMGECxcXFbts3bNjAVVddRWhoKAEBAQwYMIDNmze7tVEUhQ8++ICrr74ab29v2rVrx5o1a9i3bx8DBw7Ex8eHvn37sn///rr+uHVGAtDFFN7e9Xw02aNlCCGEcKeqKoU2+0V/qKpa41rnz59P27ZtadOmDbfddhsff/yxdpz58+czefJkpk6dysaNG4mKiuK9995z2z8vL4+xY8eycuVK1q5dS+vWrRk+fDh5eXlu7f7xj38wZswYkpOTadu2Lbfccgv33HMPkyZNYuPGjaiqyoMPPnj+X7qHeXQixFdeeYVvvvmGXbt24eXlRd++fXn11Vdp06bNWff76quveOGFF0hNTaV169a8+uqrDB8+XNuuqiovvfQSH330EdnZ2fTr14/333+f1q1b1/VHOrvorq7n9K3gdIJO8qcQQtQHRSUO2r/460V/37/+noS3qWa/imfPns1tt90GwNChQ8nJyeH3339n4MCBTJ8+nQkTJjBhwgQAXn75ZZYsWeLWCzRo0CC343344YcEBgby+++/c/XVV2vrx48fz4033gjA008/TZ8+fXjhhRdISkoC4JFHHmH8+PE1/9D1hEd/A//+++888MADrF27lsWLF1NSUsKQIUMoKCg44z6rV6/m5ptvZsKECWzZsoVRo0YxatQotm/frrWZNm0a77zzDrNmzWLdunX4+PiQlJRUqRvwogu9BIzeYMuDkw2321AIIYRn7N69m/Xr13PzzTcDYDAYGD16NLNnzwZg586dJCYmuu3Tp08ft+XMzEzuuusuWrduTUBAAP7+/uTn55OWlubWrlOnTtrriIgIAC699FK3dcXFxeTm5tbeB7yIPNoDtHDhQrflOXPmEB4ezqZNm7j88sur3Oftt99m6NChPPnkk4Cri27x4sXMmDGDWbNmoaoq06dP5/nnn+faa68FYO7cuURERLBgwQJuuummuv1QZ6M3QOSlcGgdHN0CoR7ukRJCCAGAl1HPX39P8sj71sTs2bOx2+1ER0dr61RVxWw2M2PGjGodY+zYsZw4cYK3336b2NhYzGYzffr0wWazubUzGo3a67LJB6ta53Q6a/QZ6ot6dQ4mJycHgODg4DO2WbNmDYMHD3Zbl5SUxJo1awBISUkhIyPDrU1AQACJiYlam9NZrVZyc3PdHnWm7DTY0S119x5CCCFqRFEUvE2Gi/6oyazGdruduXPn8sYbb5CcnKw9tm7dSnR0NF988QXt2rVj3bp1bvutXbvWbXnVqlU8/PDDDB8+nA4dOmA2mzl+/HitfI8NSb25GarT6eTRRx+lX79+dOzY8YztMjIytK64MhEREWRkZGjby9adqc3pXnnlFaZMmXIh5VefBCAhhBDn4ccff+TUqVNMmDCBgIAAt23XX389s2fP5oknnmDcuHH06NGDfv368d///pcdO3bQqlUrrW3r1q35z3/+Q48ePcjNzeXJJ5/UbjLalNSbHqAHHniA7du38+WXX1709540aRI5OTna49ChQ3X3ZtpA6D/B6ai79xFCCNGozJ49m8GDB1cKP+AKQBs3bqRdu3a88MILPPXUU3Tv3p2DBw9y3333VTrOqVOn6NatG7fffjsPP/ww4eHhF+tj1Bv1ogfowQcf5Mcff2TFihU0b978rG0jIyPJzMx0W5eZmUlkZKS2vWxdVFSUW5szTQVuNpsxm80X8AlqICQBjD5QUgDH90J424vzvkIIIRq0H3744YzbevXqpV0K36lTJ5599lm37a+++qr2umvXrmzYsMFt+//93/+5LZ9+eX7Lli0rrRs4cOB5XcZfX3i0B6hsDoFvv/2WZcuWERcXd859+vTpw9KlS93WLV68WBvlHhcXR2RkpFub3Nxc1q1bV2kkvEfo9BDV2fVaToMJIYQQHuHRAPTAAw/w2Wef8fnnn+Pn50dGRgYZGRkUFRVpbcaMGcOkSZO05UceeYSFCxfyxhtvsGvXLiZPnszGjRu1yZgUReHRRx/l5Zdf5vvvv2fbtm2MGTOG6OhoRo0adbE/YtW002DJHi1DCCGEaKo8egrs/fddt4QYOHCg2/pPPvmEcePGAZCWloauwoSBffv25fPPP+f555/n2WefpXXr1ixYsMBt4PRTTz1FQUEBd999N9nZ2fTv35+FCxdisVjq/DNVS3QX17P0AAkhhBAeoagN+QReHcnNzSUgIICcnBz8/f1r/w2O74UZPcDgBZMOu+YHEkIIcVEUFxeTkpJCXFxc/fmPsai2s/38avL7u95cBdakBMeDyQ/sRXB8t6erEUIIIZocCUCeoNNVOA2W7MlKhBBCiCZJApCnyJVgQgghhMdIAPIUmRFaCCGE8BgJQJ5SFoAyt4OjxLO1CCGEEE2MBCBPCW4F5gCwF8OxXZ6uRgghhGhSJAB5iqJAtIwDEkIIUX3Hjh3jvvvuo0WLFpjNZiIjI0lKSmLVqlVamy1btjB69GiioqIwm83ExsZy9dVX88MPP2i3rkhNTUVRFO3h5+dHhw4deOCBB9i7d6+nPt5FJQHIk7RxQMkeLUMIIUTDcP3117NlyxY+/fRT9uzZw/fff8/AgQM5ceIEAN999x29e/cmPz+fTz/9lJ07d7Jw4UKuu+46nn/+eXJyctyOt2TJEtLT09m6dStTp05l586ddO7cudItpxojmYHPk6K6uJ6lB0gIIcQ5ZGdn88cff7B8+XIGDBgAQGxsLL169QKgoKCACRMmMGLECL755hu3fdu1a8eECRMq3bw0JCREu4l4q1atGDlyJFdeeSUTJkxg//796PX6i/DJPEN6gDyp4kBou82ztQghRFOmqmAruPiPGtyMwdfXF19fXxYsWIDVaq20fdGiRZw4cYKnnnrqjMdQFOWs76HT6XjkkUc4ePAgmzZtqnZtDZH0AHlSUEuwBEJxNmT9VT45ohBCiIurpBCmRl/89332KJh8qtXUYDAwZ84c7rrrLmbNmkW3bt0YMGAAN910E506dWLPnj0AtGnTRttnw4YNXHHFFdryl19+ydVXX33W92nbti3gGidU1rvUGEkPkCcpitwZXgghRLVdf/31HD16lO+//56hQ4eyfPlyunXrxpw5c6ps36lTJ5KTk0lOTqagoAC73X7O9yg7TXau3qKGTnqAPC26Cxz4zTUOqPs4T1cjhBBNk9Hb1RvjifetIYvFwlVXXcVVV13FCy+8wJ133slLL73EW2+9BcDu3bvp3bs3AGazmYSEhBodf+fOnQDExcXVuLaGRAKQp8mM0EII4XmKUu1TUfVN+/btWbBgAUOGDCE4OJhXX32Vb7/99ryO5XQ6eeedd4iLi6Nr1661XGn9IgHI07SB0H+B3QoGs2frEUIIUS+dOHGCG264gTvuuINOnTrh5+fHxo0bmTZtGtdeey2+vr78+9//ZvTo0YwYMYKHH36Y1q1bk5+fz8KFCwEqXdV14sQJMjIyKCwsZPv27UyfPp3169fz008/NeorwEACkOcFxIBXMBSdhMwd0KybpysSQghRD/n6+pKYmMhbb73F/v37KSkpISYmhrvuuotnn30WgOuuu47Vq1fz6quvMmbMGE6ePElAQAA9evSocgD04MGDAfD29iY2NpYrrriCDz/8sManzRoiRT19UgBBbm4uAQEB5OTk4O/vX/dv+J+/wf6lMOJN6Dmh7t9PCCGasOLiYlJSUoiLi8NisXi6HFFDZ/v51eT3t1wFVh/IOCAhhBDiopIAVB/IpfBCCCHERSUBqD4omwAxayeUFHm0FCGEEKIpkABUH/g3A58wcNpdA6GFEEIIUackANUHFWeElnFAQgghRJ2TAFRfaHeGT/ZkFUIIIUSTIAGovpAeICGEEOKikQBUX5QFoGM7wVbo2VqEEEKIRk4CUH3hHwW+kaA6IXO7p6sRQgghGjUJQPVJ2eXwchpMCCFELZk8eTJdunTxdBlnNXDgQB599NGL+p4SgOoTGQckhBCiGtasWYNer2fEiBGeLqXBkgBUn2gBKNmjZQghhKjfZs+ezUMPPcSKFSs4evSop8sBoKSkxNMl1IgEoPqk7FL447vBmu/RUoQQQtRP+fn5zJs3j/vuu48RI0YwZ84ct+3/+te/iIiIwM/PjwkTJlBcXOy2fcOGDVx11VWEhoYSEBDAgAED2Lx5s1ubXbt20b9/fywWC+3bt2fJkiUoisKCBQsASE1NRVEU5s2bx4ABA7BYLPz3v//lxIkT3HzzzTRr1gxvb28uvfRSvvjiC7djFxQUMGbMGHx9fYmKiuKNN96o9e+oOjwagFasWMHIkSOJjo52+2LPZNy4cSiKUunRoUMHrc3kyZMrbW/btm0df5Ja4hcBftGugdAZ2zxdjRBCNBmqqlJYUnjRH6qq1rjW+fPn07ZtW9q0acNtt93Gxx9/rB1n/vz5TJ48malTp7Jx40aioqJ477333PbPy8tj7NixrFy5krVr19K6dWuGDx9OXl4eAA6Hg1GjRuHt7c26dev48MMPee6556qs5ZlnnuGRRx5h586dJCUlUVxcTPfu3fnpp5/Yvn07d999N7fffjvr16/X9nnyySf5/fff+e6771i0aBHLly+vFMAuBsNFf8cKCgoK6Ny5M3fccQd/+9vfztn+7bff5l//+pe2bLfb6dy5MzfccINbuw4dOrBkyRJt2WDw6MesmeiusPuoaxxQbB9PVyOEEE1Ckb2IxM8TL/r7rrtlHd5G7xrtM3v2bG677TYAhg4dSk5ODr///jsDBw5k+vTpTJgwgQkTJgDw8ssvs2TJErdeoEGDBrkd78MPPyQwMJDff/+dq6++msWLF7N//36WL19OZGQkAP/85z+56qqrKtXy6KOPVvr9/cQTT2ivH3roIX799Vfmz59Pr169yM/PZ/bs2Xz22WdceeWVAHz66ac0b968Rt9BbfBoMhg2bBjDhg2rdvuAgAACAgK05QULFnDq1CnGjx/v1s5gMGg/tAYnuivs/knuDC+EEKKS3bt3s379er799lvA9ftu9OjRzJ49m4EDB7Jz507uvfdet3369OnDb7/9pi1nZmby/PPPs3z5crKysnA4HBQWFpKWlqa9R0xMjNvv0V69elVZT48ePdyWHQ4HU6dOZf78+Rw5cgSbzYbVasXb2xXy9u/fj81mIzGxPGwGBwfTpk2bC/hWzk8D6hqpbPbs2QwePJjY2Fi39Xv37iU6OhqLxUKfPn145ZVXaNGihYeqrCG5FF4IIS46L4MX625Z55H3rYnZs2djt9uJjo7W1qmqitlsZsaMGdU6xtixYzlx4gRvv/02sbGxmM1m+vTpg81mq1EtAD4+Pm7Lr732Gm+//TbTp0/n0ksvxcfHh0cfffS8jl3XGmwAOnr0KL/88guff/652/rExETmzJlDmzZtSE9PZ8qUKVx22WVs374dPz+/Ko9ltVqxWq3acm5ubp3WflbaQOi9YM0Dc9U1CyGEqD2KotT4VNTFZrfbmTt3Lm+88QZDhgxx2zZq1Ci++OIL2rVrx7p16xgzZoy2be3atW5tV61axXvvvcfw4cMBOHToEMePH9e2t2nThkOHDpGZmUlERATgGjhdHatWreLaa6/VTtE5nU727NlD+/btAYiPj8doNLJu3TqtY+LUqVPs2bOHAQMG1OTruGANNgB9+umnBAYGMmrUKLf1FU+pderUicTERGJjY5k/f752TvR0r7zyClOmTKnLcqvPNwwCYiDnEKT/CS37eboiIYQQ9cCPP/7IqVOnmDBhgttwEIDrr7+e2bNn88QTTzBu3Dh69OhBv379+O9//8uOHTto1aqV1rZ169b85z//oUePHuTm5vLkk0/i5VXeE3XVVVcRHx/P2LFjmTZtGnl5eTz//POAKyieTevWrfnf//7H6tWrCQoK4s033yQzM1MLQL6+vkyYMIEnn3ySkJAQwsPDee6559DpLv41WQ3yMnhVVfn444+5/fbbMZlMZ20bGBjIJZdcwr59+87YZtKkSeTk5GiPQ4cO1XbJNRPV2fUsp8GEEEKUKhv2cXr4AVcA2rhxI+3ateOFF17gqaeeonv37hw8eJD77ruv0nFOnTpFt27duP3223n44YcJDw/Xtuv1ehYsWEB+fj49e/bkzjvv1K4Cs1gsZ63x+eefp1u3biQlJTFw4EAiIyMrdVS89tprXHbZZYwcOZLBgwfTv39/unfvfp7fyvlT1PO5Bq8OKIrCt99+W+mLqsry5cu54oor2LZtGx07djxr2/z8fFq0aMHkyZN5+OGHq1VLbm4uAQEB5OTk4O/vX619atWK12HZP6Dj/8H/zb747y+EEI1YcXExKSkpxMXFnfMXunBZtWoV/fv3Z9++fcTHx3u0lrP9/Gry+9ujp8Dy8/PdemZSUlJITk4mODiYFi1aMGnSJI4cOcLcuXPd9ps9ezaJiYlVhp8nnniCkSNHEhsby9GjR3nppZfQ6/XcfPPNdf55ao3cEkMIIYQHffvtt/j6+tK6dWv27dvHI488Qr9+/TwefmqTRwPQxo0bueKKK7TliRMnAq4R6nPmzCE9PV27LK9MTk4OX3/9NW+//XaVxzx8+DA333wzJ06cICwsjP79+7N27VrCwsLq7oPUtrIAdHI/FOeApXJ3pxBCCFFX8vLyePrpp0lLSyM0NJTBgwd7bMbmulJvToHVJx4/BQYw/VLIToOxP0Dc5Z6pQQghGiE5Bdaw1dYpsAY5CLpJkNNgQgghRJ2RAFRfyZ3hhRBCiDojAai+KpsQUXqAhBBCiFonAai+KrslxqkUKDrl0VKEEEKIxkYCUH3lFQRBca7XchpMCCGEqFUSgOqzsnFAcmd4IYQQolZJAKrP5M7wQgghRJ2QAFSfyaXwQgghKhg3bhyKoqAoCkajkbi4OJ566imKi4u1NmXbT78LvNVqJSQkBEVRWL58ubb+999/Z9CgQQQHB+Pt7U3r1q0ZO3YsNpvtYn0sj5AAVJ+V3RQ1Ow0KT3q2FiGEEPXC0KFDSU9P58CBA7z11lt88MEHvPTSS25tYmJi+OSTT9zWld3eoqK//vqLoUOH0qNHD1asWMG2bdt49913MZlMOByO866xIYQnCUD1mSUAgkvvuyK9QEIIIQCz2UxkZCQxMTGMGjWKwYMHs3jxYrc2Y8eO5csvv6SoqEhb9/HHHzN27Fi3dosWLSIyMpJp06bRsWNH4uPjGTp0KB999BFeXl4AzJkzh8DAQBYsWEDr1q2xWCwkJSVx6NAh7TiTJ0+mS5cu/Pvf/3aboTktLY1rr70WX19f/P39ufHGG8nMzKy03wcffEBMTAze3t7ceOON5OTk1Pr3djoJQPWdnAYTQoiLxmm1nvGhntarcaFta8P27dtZvXo1JpPJbX337t1p2bIlX3/9NeAKIitWrOD22293axcZGUl6ejorVqw46/sUFhbyz3/+k7lz57Jq1Sqys7O56aab3Nrs27ePr7/+mm+++Ybk5GScTifXXnstJ0+e5Pfff2fx4sUcOHCA0aNHV9pv/vz5/PDDDyxcuJAtW7Zw//33n+9XUm0evRmqqIborrD9fxKAhBDiIjjyyKNn3Gbp2JGwBx/Qlo8++VSloFPG3Lo14Y9P1JbTn3seZ36+W5uYWe+fV40//vgjvr6+2O12rFYrOp2OGTNmVGp3xx138PHHH3PbbbcxZ84chg8fXunG4DfccAO//vorAwYMIDIykt69e3PllVcyZswYt3tplZSUMGPGDBITEwH49NNPadeuHevXr6dXr16A67TX3LlztfdYvHgx27ZtIyUlhZiYGADmzp1Lhw4d2LBhAz179gRc9/aaO3cuzZo1A+Ddd99lxIgRvPHGG0RGRp7Xd1Qd0gNU32mXwm/1bB1CCCHqhSuuuILk5GTWrVvH2LFjGT9+PNdff32ldrfddhtr1qzhwIEDzJkzhzvuuKNSG71ezyeffMLhw4eZNm0azZo1Y+rUqXTo0IH09HStncFg0AILQNu2bQkMDGTnzp3autjYWLeAtXPnTmJiYrTwA9C+fftK+7Vo0UILPwB9+vTB6XSye/fu8/h2qk96gOq7qE6AAjmHIP8Y+IadcxchhBDnp9nb08+4TVEUt+Xo16ZVu23UP1++oLoq8vHxISEhAXCN6+ncuTOzZ89mwoQJbu1CQkK4+uqrmTBhAsXFxQwbNoy8vLwqj9msWTNuv/12br/9dv7xj39wySWXMGvWLKZMmVKjuhoS6QGq78x+ENra9VomRBRCiDqlM5vP+FBOG2dzoW1rpV6djmeffZbnn3/ebcBzmTvuuIPly5czZswY9Hp9tY4ZFBREVFQUBQUF2jq73c7GjRu15d27d5OdnU27du3OeJx27dpx6NAht8HSf/31F9nZ2bRv315bl5aWxtGjR7XltWvXotPpaNOmTbXqPV8SgBoCuTO8EEKIM7jhhhvQ6/XMnDmz0rahQ4dy7Ngx/v73v1e57wcffMB9993HokWL2L9/Pzt27ODpp59mx44djBw5UmtnNBp56KGHWLduHZs2bWLcuHH07t1bG/9TlcGDB3PppZdy6623snnzZtavX8+YMWMYMGAAPXr00NpZLBbGjh3L1q1b+eOPP3j44Ye58cYb63T8D0gAahjkzvBCCCHOwGAw8OCDDzJt2jS3XhtwnYoLDQ2tdJVYmV69epGfn8+9995Lhw4dGDBgAGvXrmXBggUMGDBAa+ft7c3TTz/NLbfcQr9+/fD19WXevHlnrUtRFL777juCgoK4/PLLGTx4MK1ataq0X0JCAn/7298YPnw4Q4YMoVOnTrz33nvn+W1Un6Kqqlrn79LA5ObmEhAQQE5OjtsoeI85uAY+GQp+0fD4znO3F0IIcUbFxcWkpKS4zVcjzmzOnDk8+uijZGdn1/qxJ0+ezIIFC0hOTq72Pmf7+dXk97f0ADUEkZeCooO8o5CXee72QgghhDgrCUANgdkXQi9xvZaB0EIIIcQFkwDUUMiM0EIIITxg3LhxdXL6C1ynwGpy+qs2SQBqKCQACSGEELVGAlBDIZfCCyGEELVGAlBDEdHRNRA6PwNy08/dXgghhBBnJAGooTB5Q1jpjJtyGkwIIYS4IBKAGhLtxqjJHi1DCCGEaOgkADUk0V1cz9IDJIQQQlwQCUANScUrwWQCbyGEEOK8SQBqSCI6gM4ABccg94inqxFCCHGRjRs3jlGjRlWr7cCBA1EURXtERERwww03cPDgQa1NamqqW5uyx2233eZ2rK+//ppBgwYRFBSEl5cXbdq04Y477mDLlvIzEg6Hg3/961+0bdsWLy8vgoODSUxM5N///netfPbaJgGoITF6QXjZQOhkj5YihBCi/rvrrrtIT0/n6NGjfPfddxw6dKhSuAFYsmQJ6enp2qPineWffvppRo8eTZcuXfj+++/ZvXs3n3/+Oa1atWLSpElauylTpvDWW2/xj3/8g7/++ovffvuNu+++u84mUbxQBk8XIGooqgtkbHOdBmt3taerEUIIUY95e3sTGRkJQFRUFA8++CD33HNPpXYhISFau4rWrl3LtGnTePvtt3n44Ye19S1atKB79+5UvJ/6999/z/33388NN9ygrevcuXNtfpxa5dEeoBUrVjBy5Eiio6NRFIUFCxactf3y5cur7KrLyMhwazdz5kxatmyJxWIhMTGR9evX1+GnuMhkRmghhKgz9hIH9hKH2y92p8OJvcSBw+Gsflv7udtebCdPnmT+/PkkJiZWe58vvvgCX19f7r///iq3K4qivY6MjGTZsmUcO3bsgmu9GDwagAoKCujcubNbV1t17N69262rLjw8XNs2b948Jk6cyEsvvcTmzZvp3LkzSUlJZGVl1Xb5nlHxUngZCC2EELXq1w+38+uH27EVO7R1+7cc49cPt7NjhfvYyyUf/8WvH26nKK9EW5e67QS/fridP5cddmu7bO4ufv1wO/mnrHX7AU7z3nvv4evri4+PDyEhIezevZuPP/64Uru+ffvi6+urPcrG9uzZs4dWrVphMJSfMHrzzTfd2ubk5Gjrjx07RmRkJJ06deLee+/ll19+uTgf9Dx4NAANGzaMl19+meuuu65G+4WHhxMZGak9dLryj/Hmm29y1113MX78eNq3b8+sWbPw9vau8gfeIEV0AJ0RCk9AziFPVyOEEKIe6NChgxZIhg0bpq2/9dZbSU5OZuvWraxcuZKEhASGDBlCXl6e2/7z5s0jOTlZe7Rv3/6M73XHHXeQnJzMBx98QEFBgdar1b59e7Zv387atWu54447yMrKYuTIkdx5551186EvUIMcA9SlSxesVisdO3Zk8uTJ9OvXDwCbzcamTZvcBmXpdDoGDx7MmjVrzng8q9WK1VqeynNzc+uu+AtlMENEe0jf6joNFtjC0xUJIUSjkXR3RwD0hvL/WMd3DSOucyiKTnFrO/iO9pXatrw0hBYdgt1ODQEMGtO2Utva9PPPP1NS4uqJ8vLy0tYHBASQkJAAQEJCArNnzyYqKop58+a5BZOYmBitXUWtW7dm5cqVlJSUYDQaAQgMDCQwMJDDhw9Xaq/T6ejZsyc9e/bk0Ucf5bPPPuP222/nueeeIy4urlY/84VqUFeBRUVFMWvWLL7++mu+/vprYmJiGDhwIJs3bwbg+PHjOBwOIiIi3PaLiIioNE6ooldeeYWAgADtERMTU6ef44LJOCAhhKgTBqMeg1HvFmB0eh0Gox69Xlf9toZzt61NsbGxJCQkkJCQQLNmzc7YTq/XA1BUVFSt4958883k5+fz3nvvnVddZT1JBQUF57V/XWpQPUBt2rShTZs22nLfvn3Zv38/b731Fv/5z3/O+7iTJk1i4sSJ2nJubm79DkHRXWHTHLkUXgghmqCcnBySk5Pd1oWEhFT5e6uwsFDrAMjMzOQf//gHFouFIUOGVOu9+vTpw+OPP87jjz/OwYMH+dvf/kZMTAzp6enMnj0bRVG0YSj/93//R79+/ejbty+RkZGkpKQwadIkLrnkEtq2bXthH7oONKgAVJVevXqxcuVKAEJDQ9Hr9WRmZrq1yczMrPLyvjJmsxmz2VynddaqqC6u57IZoevofxRCCCHqn+XLl9O1a1e3dRMmTKhywsGPPvqIjz76CICgoCA6derEzz//7NaZcC6vv/46vXr14v333+fjjz+msLCQiIgILr/8ctasWYO/vz8ASUlJfPHFF7zyyivk5OQQGRnJoEGDmDx5stsg6vqi/lVUQ8nJyURFRQFgMpno3r07S5cu1WbKdDqdLF26lAcffNCDVday8PagN0FxNpxKheD6dV5VCCFE3ZgzZw5z5sypVtvly5efs03Lli2rdWn+jTfeyI033njWNnfddRd33XVXtWqrDzwagPLz89m3b5+2nJKSQnJyMsHBwbRo0YJJkyZx5MgR5s6dC8D06dOJi4ujQ4cOFBcX8+9//5tly5axaNEi7RgTJ05k7Nix9OjRg169ejF9+nQKCgoYP378Rf98dcZggoiOcHSz63J4CUBCCCFEjXg0AG3cuJErrrhCWy4bhzN27FjmzJlDeno6aWlp2nabzcbjjz/OkSNH8Pb2plOnTixZssTtGKNHj+bYsWO8+OKLZGRk0KVLFxYuXFhpYHSDF93FFYCOboEONZtGQAghhGjqFNWT01LWU7m5uQQEBJCTk6Od26x3Ns+F7x+CuMth7A+erkYIIRqM4uJiUlJSiIuLw2KxeLocUUNn+/nV5Pd3g7oMXlSgXQq/VWaEFkIIIWpIAlBDFdYWDBaw5sDJA56uRgghGhw5AdIw1dbPTQJQQ6U3ugZCg0yIKIQQNVA2o3FhYaGHKxHno+znVvZzPF8N/jL4Ji26KxzZ6ApAl/6fp6sRQogGQa/XExgYqN0k29vbu85maBa1R1VVCgsLycrKIjAwUJvV+nxJAGrItDvDb/VsHUII0cCUTY5bFoJEwxEYGHjWyY2rSwJQQxbdxfV8NBmcTtDJGU0hhKgORVGIiooiPDxcu4moqP+MRuMF9/yUkQDUkIW2AYMX2PLg5H4Ibe3pioQQokHR6/W19gtVNCzSZdCQ6Q0Q1cn1WgZCCyGEENUmAaih0+YDSvZoGUIIIURDIgGooat4Z3ghhBBCVIsEoIau4pVgTodnaxFCCCEaCAlADV1oazD6QEkBnNjn6WqEEEKIBkECUEOn08tAaCGEEKKGJAA1BtpAaAlAQgghRHVIAGoMJAAJIYQQNSIBqDEoC0AZ28Bh92wtQgghRAMgAagxCI4Hkx+UFMLxPZ6uRgghhKj3JAA1BjodRHV2vZbTYEIIIcQ5SQBqLMpujJqe7MkqhBBCiAZBAlBjIQOhhRBCiGqTANRYyEBoIYQQotokADUWQXFgDgB7MRzb5elqhBBCiHpNAlBjodNBdOlA6MPrPVuLEEIIUc9JAGpMWvR1PR9c7dk6hBBCiHpOAlBj0rKf6zl1JaiqZ2sRQggh6jEJQI1J856gN0FeOpw84OlqhBBCiHpLAlBjYvSCZj1crw+u8mwtQgghRD0mAaix0U6DSQASQgghzkQCUGMTK+OAhBBCiHORANTYxPQCnRFyD0P2QU9XI4QQQtRLNQ5An376KT/99JO2/NRTTxEYGEjfvn05eLBmv3BXrFjByJEjiY6ORlEUFixYcNb233zzDVdddRVhYWH4+/vTp08ffv31V7c2kydPRlEUt0fbtm1rVFeDZvKBZt1cr+U0mBBCCFGlGgegqVOn4uXlBcCaNWuYOXMm06ZNIzQ0lMcee6xGxyooKKBz587MnDmzWu1XrFjBVVddxc8//8ymTZu44oorGDlyJFu2uN//qkOHDqSnp2uPlStX1qiuBq/iaTAhhBBCVGKo6Q6HDh0iISEBgAULFnD99ddz9913069fPwYOHFijYw0bNoxhw4ZVu/306dPdlqdOncp3333HDz/8QNeuXbX1BoOByMjIGtXSqLTsByvfhIMSgIQQQoiq1LgHyNfXlxMnTgCwaNEirrrqKgAsFgtFRUW1W905OJ1O8vLyCA4Odlu/d+9eoqOjadWqFbfeeitpaWlnPY7VaiU3N9ft0aDF9AZFD9lpkH3I09UIIYQQ9U6NA9BVV13FnXfeyZ133smePXsYPnw4ADt27KBly5a1Xd9Zvf766+Tn53PjjTdq6xITE5kzZw4LFy7k/fffJyUlhcsuu4y8vLwzHueVV14hICBAe8TExFyM8uuO2bf87vAyH5AQQghRSY0D0MyZM+nTpw/Hjh3j66+/JiQkBIBNmzZx880313qBZ/L5558zZcoU5s+fT3h4uLZ+2LBh3HDDDXTq1ImkpCR+/vlnsrOzmT9//hmPNWnSJHJycrTHoUONoNdEmw/oD8/WIYQQQtRDNR4DFBgYyIwZMyqtnzJlSq0UVB1ffvkld955J1999RWDBw8+a9vAwEAuueQS9u3bd8Y2ZrMZs9lc22V6Vmx/WPW2XAkmhBBCVKHGPUALFy50u6pq5syZdOnShVtuuYVTp07VanFV+eKLLxg/fjxffPEFI0aMOGf7/Px89u/fT1RUVJ3XVq+06A2KDk6lQO5RT1cjhBBC1Cs1DkBPPvmkNkh427ZtPP744wwfPpyUlBQmTpxYo2Pl5+eTnJxMcnIyACkpKSQnJ2uDlidNmsSYMWO09p9//jljxozhjTfeIDExkYyMDDIyMsjJydHaPPHEE/z++++kpqayevVqrrvuOvR6/UU9PVcvWPwhqrPrtfQCCSGEEG5qHIBSUlJo3749AF9//TVXX301U6dOZebMmfzyyy81OtbGjRvp2rWrdgn7xIkT6dq1Ky+++CIA6enpbldwffjhh9jtdh544AGioqK0xyOPPKK1OXz4MDfffDNt2rThxhtvJCQkhLVr1xIWFlbTj9rwxco4ICGEEKIqNR4DZDKZKCwsBGDJkiVaD01wcHCNLx8fOHAg6lnuVzVnzhy35eXLl5/zmF9++WWNamjUWl4Ga2bIlWBCCCHEaWocgPr378/EiRPp168f69evZ968eQDs2bOH5s2b13qB4gK06A0ocGIf5GWAXxOeHFIIIYSooManwGbMmIHBYOB///sf77//Ps2aNQPgl19+YejQobVeoLgAXoEQeanrtfQCCSGEEJoa9wC1aNGCH3/8sdL6t956q1YKErWsZX/I+NN1X7CO13u6GiGEEKJeqHEAAnA4HCxYsICdO3cCrpuPXnPNNej1+lotTtSClv1h7XtyJZgQQghRQY0D0L59+xg+fDhHjhyhTZs2gOtWEjExMfz000/Ex8fXepHiArToAyhwfDfkHwPfJng1nBBCCHGaGo8Bevjhh4mPj+fQoUNs3ryZzZs3k5aWRlxcHA8//HBd1CguhHcwRHRwvZa7wwshhBDAefQA/f7776xdu9btDuwhISH861//ol+/frVanKglsf0gc7vrNFiH6zxdjRBCCOFxNe4BMpvNVd5ZPT8/H5PJVCtFiVrWsr/rWa4EE0IIIYDzCEBXX301d999N+vWrUNVVVRVZe3atdx7771cc801dVGjuFBlM0Jn/QUFJzxbixBCCFEP1DgAvfPOO8THx9OnTx8sFgsWi4V+/fqRkJDA9OnT66BEccF8QiCsneu19AIJIYQQNR8DFBgYyHfffce+ffu0y+DbtWtHQkJCrRcnalHL/nBspysAtZeeOiGEEE3bec0DBJCQkOAWev7880969OiBzWarlcJELWvZDzZ8JPMBCSGEEJzHKbAzUVUVh8NRW4cTta1sHFDmdig86dlahBBCCA+rtQAk6jnfcAi9BFAhbY2nqxFCCCE8SgJQU1J2ObycBhNCCNHEVXsMUG5u7lm3VzU3kKhnYvvBxo9lRmghhBBNXrUDUGBgIIqinHG7qqpn3S7qgbIeoIxtUJQNXoGerEYIIYTwmGoHoN9++60u6xAXg18kBMfDyf2QthbaDPV0RUIIIYRHVDsADRgwoC7rEBdLy/6uAHRwpQQgIYQQTZYMgm5qZCC0EEIIIQGoySmbDyg9GYrPPrBdCCGEaKwkADU1Ac0gqCWoTji0ztPVCCGEEB4hAagp0k6DyeXwQgghmiYJQE1RbGkAkjvDCyGEaKJqfDPU6667rsr5fhRFwWKxkJCQwC233EKbNm1qpUBRB1qWjgM6shms+WD29Ww9QgghxEVW4x6ggIAAli1bxubNm1EUBUVR2LJlC8uWLcNutzNv3jw6d+7MqlXSu1BvBbZwPVSHjAMSQgjRJNU4AEVGRnLLLbdw4MABvv76a77++mv279/PbbfdRnx8PDt37mTs2LE8/fTTdVGvqC1yGkwIIUQTVuMANHv2bB599FF0uvJddTodDz30EB9++CGKovDggw+yffv2Wi1U1LKy02AyH5AQQogmqMYByG63s2vXrkrrd+3ahcPhAMBisch9weq7svmAjmwCW6FnaxFCCCEushoPgr799tuZMGECzz77LD179gRgw4YNTJ06lTFjxgDw+++/06FDh9qtVNSuoJbg3xxyD8Ph9dBqoKcrEkIIIS6aGgegt956i4iICKZNm0ZmZiYAERERPPbYY9q4nyFDhjB0qNxnql5TFNdpsD/nuU6DSQASQgjRhNT4FJher+e5554jPT2d7OxssrOzSU9P59lnn0Wv1wPQokULmjdvfs5jrVixgpEjRxIdHY2iKCxYsOCc+yxfvpxu3bphNptJSEhgzpw5ldrMnDmTli1bYrFYSExMZP369TX9mE1D2WkwGQgthBCiibmgiRD9/f3x9/c/7/0LCgro3LkzM2fOrFb7lJQURowYwRVXXEFycjKPPvood955J7/++qvWZt68eUycOJGXXnqJzZs307lzZ5KSksjKyjrvOhutshmhD2+AkiLP1iKEEEJcRIqqqmpNdsjMzOSJJ55g6dKlZGVlcfruZQOha1yIovDtt98yatSoM7Z5+umn+emnn9yuMLvpppvIzs5m4cKFACQmJtKzZ09mzJgBgNPpJCYmhoceeohnnnmmWrXk5uYSEBBATk7OBQW8ek9V4c12kJcOY3+EuMs8XZEQQghx3mry+7vGY4DGjRtHWloaL7zwAlFRURf1aq81a9YwePBgt3VJSUk8+uijANhsNjZt2sSkSZO07TqdjsGDB7NmzZozHtdqtWK1WrXl3Nwmcpd0RXGdBtv+P9dpMAlAQgghmogaB6CVK1fyxx9/0KVLlzoo5+wyMjKIiIhwWxcREUFubi5FRUWcOnUKh8NRZZuqLt0v88orrzBlypQ6qbnea1kagOTGqEIIIZqQGo8BiomJqXTaq6GbNGkSOTk52uPQoUOeLuniaVna63N4A9itZ28rhBBCNBI1DkDTp0/nmWeeITU1tQ7KObvIyEjt0vsymZmZ+Pv74+XlRWhoKHq9vso2kZGRZzyu2WzWBnRf6MDuBickAXzCwV7smhRRCCGEaAJqHIBGjx7N8uXLiY+Px8/Pj+DgYLdHXerTpw9Lly51W7d48WL69OkDgMlkonv37m5tnE4nS5cu1dqI05TNBwRyWwwhhBBNRo3HAE2fPr3W3jw/P599+/ZpyykpKSQnJxMcHEyLFi2YNGkSR44cYe7cuQDce++9zJgxg6eeeoo77riDZcuWMX/+fH766SftGBMnTmTs2LH06NGDXr16MX36dAoKChg/fnyt1d3oxPaDHd9C6h8w4ElPVyOEEELUuRoHoLFjx9bam2/cuJErrrhCW544caL2HnPmzCE9PZ20tDRte1xcHD/99BOPPfYYb7/9Ns2bN+ff//43SUlJWpvRo0dz7NgxXnzxRTIyMujSpQsLFy6sNDBaVFA2DujQerDbwGDybD1CCCFEHavWPEC5ubnauJhzXSLeGMbPNJl5gMqoKryWAIXH4Y5F0CLR0xUJIYQQNVbr8wAFBQWRnp5OeHg4gYGBVc79o6oqiqKc90SIwoMUBWL7ws7v4eBKCUBCCCEavWoFoGXLlmkDnH/77bc6LUh4SMv+rgCUuhIue9zT1QghhBB1qloBaMCAAVW+Fo1I2X3B0taBowT0Rs/WI4QQQtShGg+CBsjOzmb9+vVkZWXhdDrdto0ZM6ZWChMXWVg78AqColOQvhWa9/B0RUIIIUSdqXEA+uGHH7j11lvJz8/H39/fbTyQoigSgBoqnc51OfyuH12nwSQACSGEaMRqPBHi448/zh133EF+fj7Z2dmcOnVKe5w8ebIuahQXS2zZhIhyXzAhhBCNW40D0JEjR3j44Yfx9vaui3qEJ2njgNaCw+7ZWoQQQog6VOMAlJSUxMaNG+uiFuFpER3AEgC2PMj409PVCCGEEHWmxmOARowYwZNPPslff/3FpZdeitHofrXQNddcU2vFiYtMp4cWfWHPL67TYM26eboiIYQQok5UayboinS6M3caNZaJEJvcTNAVrZ4Bi56DS4bCLfM8XY0QQghRbbU+E3RFp1/2LhqZsjvDH1wDToerV0gIIYRoZGo8Bkg0cpGdwOwP1hzI3O7paoQQQog6Ua0eoHfeeYe7774bi8XCO++8c9a2Dz/8cK0UJjxEp4cWvWHvItc4oKjOnq5ICCGEqHXVGgMUFxfHxo0bCQkJIS4u7swHUxQOHDhQqwV6QpMeAwSw6m1Y/CK0GQE3f+7paoQQQohqqfUxQCkpKVW+Fo1UbNl8QKvB6XTNEi2EEEI0IvKbTVQW1RlMvq77gmX95elqhBBCiFp3XjdDPXz4MN9//z1paWnYbDa3bW+++WatFCY8SG+AmETYv9Q1Diiyo6crEkIIIWpVjQPQ0qVLueaaa2jVqhW7du2iY8eOpKamoqoq3brJxHmNRsv+rgB0cCX0vtfT1QghhBC1qsanwCZNmsQTTzzBtm3bsFgsfP311xw6dIgBAwZwww031EWNwhPK7gt2cDXUbK5MIYQQot6rcQDauXMnY8aMAcBgMFBUVISvry9///vfefXVV2u9QOEh0V3B6A2FJ+DYLk9XI4QQQtSqGgcgHx8fbdxPVFQU+/fv17YdP3689ioTnqU3Qkwv1+vUlZ6tRQghhKhlNQ5AvXv3ZuVK1y/E4cOH8/jjj/PPf/6TO+64g969e9d6gcKDyk6DSQASQgjRyNR4EPSbb75Jfn4+AFOmTCE/P5958+bRunVruQKssSmbD+jgKtc4IEXxbD1CCCFELalRAHI4HBw+fJhOnToBrtNhs2bNqpPCRD3QrBsYLFBwDI7vgbA2nq5ICCGEqBU1OgWm1+sZMmQIp06dqqt6RH1iMMs4ICGEEI1SjccAdezYsVHc78tTcqw5ZBdne7qM6qt4GkwIIYRoJGocgF5++WWeeOIJfvzxR9LT08nNzXV7iDN7P/l9Bs4fyBe7vvB0KdXXsp/rOXWVzAckhBCi0ah2APr73/9OQUEBw4cPZ+vWrVxzzTU0b96coKAggoKCCAwMJCgoqC5rbfCifaOxO+0sOrjI06VUX7MeoDdDfgac2H/u9kIIIUQDUO1B0FOmTOHee+/lt99+q8t6GrWBMQMxKAb2Ze/jQM4BWgW08nRJ52a0QPOerltiHFwJoQmerkgIIYS4YNUOQGrp6Y8BAwbUWTGNXYA5gMToRFYdWcXi1MXc0/keT5dUPS37ucJP6iroPs7T1QghhBAXrEZjgJQ6mgdm5syZtGzZEovFQmJiIuvXrz9j24EDB6IoSqXHiBEjtDbjxo2rtH3o0KF1UntNJcUmAbD44GIPV1IDsaXjgA7KOCAhhBCNQ43mAbrkkkvOGYJOnjxZowLmzZvHxIkTmTVrFomJiUyfPp2kpCR2795NeHh4pfbffPONdisOgBMnTtC5c+dKN2IdOnQon3zyibZsNptrVFdduSLmCvSKnt2ndnMw9yCx/rGeLuncmvcEnRFyj8CpFAhuAKfuhBBCiLOoUQCaMmUKAQEBtVrAm2++yV133cX48eMBmDVrFj/99BMff/wxzzzzTKX2wcHBbstffvkl3t7elQKQ2WwmMjKyVmutDYGWQHpF9mJN+hoWH1zMnZfe6emSzs3kDc17QNoa12kwCUBCCCEauBoFoJtuuqnKXpnzZbPZ2LRpE5MmTdLW6XQ6Bg8ezJo1a6p1jNmzZ3PTTTfh4+Pjtn758uWEh4cTFBTEoEGDePnllwkJCanyGFarFavVqi3X9eX8Q1oOYU36GhalLmoYAQhcp8HS1rhOg3W73dPVCCGEEBek2mOA6mL8z/Hjx3E4HERERLitj4iIICMj45z7r1+/nu3bt3Pnne4hYujQocydO5elS5fy6quv8vvvvzNs2DAcDkeVx3nllVcICAjQHjExMef/oaphUItB6BU9O0/u5FDeoTp9r1qjzQckM0ILIYRo+KodgNR6OPh19uzZXHrppfTq1ctt/U033cQ111zDpZdeyqhRo/jxxx/ZsGEDy5cvr/I4kyZNIicnR3scOlS3oSTYEkyPyB5AAxoMHZMIehPkHILdv3i6GiGEEOKCVDsAOZ3OWj39BRAaGoperyczM9NtfWZm5jnH7xQUFPDll18yYcKEc75Pq1atCA0NZd++fVVuN5vN+Pv7uz3q2pDYIQAsTm0gAcjkA73vd73++UmwFXi2HiGEEOIC1PhWGLXJZDLRvXt3li5dqq1zOp0sXbqUPn36nHXfr776CqvVym233XbO9zl8+DAnTpwgKirqgmuuLYNaDEKn6Nh+YjtH8o94upzqGfAUBLRw9QL9/qqnqxFCCCHOm0cDEMDEiRP56KOP+PTTT9m5cyf33XcfBQUF2lVhY8aMcRskXWb27NmMGjWq0sDm/Px8nnzySdauXUtqaipLly7l2muvJSEhgaSkpIvymaoj1CuU7hHdAVhycImHq6kmkw+MeN31es1MyNzh2XqEEEKI8+TxADR69Ghef/11XnzxRbp06UJycjILFy7UBkanpaWRnp7uts/u3btZuXJllae/9Ho9f/75J9dccw2XXHIJEyZMoHv37vzxxx/1Zi6gMlfFXgXQsO4NdkkStBsJTjv8+Bg4nZ6uSAghhKgxRa2Po5s9LDc3l4CAAHJycup0PNCxwmNc+dWVqKgs/r/FRPrUv3mLqpRzBGb2Als+jHwHuo/1dEVCCCFEjX5/e7wHqCkL8w6ja3hXoAFdDQYQ0AyueM71evGLkH/Ms/UIIYQQNSQByMOGtHRdDbYotQGdBgPodTdEXgrF2bD4BU9XI4QQQtSIBCAPG9xiMADJx5LJLMg8R+t6RG+Aq98GFNj6BaSs8HRFQgghRLVJAPKwCJ8IuoR1AWBJWgO5GqxM8+7Qs3Qg+o8TwW49e3shhBCinpAAVA9oV4M1tNNgAINeAN8IOLEXVr3j6WqEEEKIapEAVA+UBaAtWVs4VtjABhR7BULSVNfrFa/Bif0eLUcIIYSoDglA9UCUbxSdQjuhorI0bem5d6hvOl4Pra4AhxV+fgJkZgUhhBD1nASgekK7GqwhTYpYRlFgxBugN8P+ZbDjG09XJIQQQpyVBKB6YnCs62qwTZmbOF503MPVnIeQeLj8CdfrhZOgKNuj5QghhBBnIwGonmjm24wOIR1wqk6WpS3zdDnnp98jEJIA+Zmw7GVPVyOEEEKckQSgeqRBnwYDMJjh6rdcrzf8G45s8mw9QgghxBlIAKpHrmrhuhpsY8ZGThaf9HA15ynucuh0E6DCD4+Cw+7pioQQQohKJADVIzH+MbQLbodDdTTc02AAQ14GSwBk/AkbPvJ0NUIIIUQlEoDqmbLTYA3q5qin8w2DwVNcr5e97Lp7vBBCCFGPSACqZ8omRVyXvo7s4mzPFnMhuo2F5r3Alg8Ln/F0NUIIIYQbCUD1TKx/LG2C2uBQHfx26DdPl3P+dDrXgGhFDzu/hz2/eroiIYQQQiMBqB7S7g3WUK8GKxPZEfrc73r98xNgK/RsPUIIIUQpCUD10FUtXQFobfpacqw5Hq7mAg2cBAExkJ0GK6Z5uhohhBACkABUL7UKaEVCYAJ2p53lh5Z7upwLY/KBYaXBZ/W7kLXTs/UIIYQQSACqt4bENoKrwcq0HQ5tRoDTDj8+Bk6npysSQgjRxEkAqqfKxgGtPrqaPFueh6upBcNeBaMPpK2B5P96uhohhBBNnASgeiohKIFWAa0ocZY0/NNgAIExcMUk1+vFL0DBCc/WI4QQokmTAFSPlfUCNYrTYACJ90HEpVB0yhWChBBCCA+RAFSPlc0KverIKvJt+R6uphboDaU3S1Vcp8FSV3q6IiGEEE2UBKB6rHVga1r6t8TmtLHi8ApPl1M7YnpC93Gu1z9OBLvNo+UIIYRomiQA1WOKojSeSRErGvwS+ITB8d2w+h1PVyOEEKIJkgBUz5WdBlt5ZCWFJY1kJmWvIEia6nq94jU4meLZeoQQQjQ5EoDquTZBbYjxi8HqsLLiSCM5DQZw6Q0QNwDsxa7bZKiqpysSQgjRhEgAqufcToOlNqLTYIoCI94EvQn2LYG/Fni6IiGEEE2IBKAGoFGeBgMITYD+E12vf3kGinM9W48QQogmo14EoJkzZ9KyZUssFguJiYmsX7/+jG3nzJmDoihuD4vF4tZGVVVefPFFoqKi8PLyYvDgwezdu7euP0adaR/cnma+zSiyF7Hq6CpPl1O7+j8GwfGQnwHLXvZ0NUIIIZoIjwegefPmMXHiRF566SU2b95M586dSUpKIisr64z7+Pv7k56erj0OHjzotn3atGm88847zJo1i3Xr1uHj40NSUhLFxcV1/XHqhKIo2r3BGtVpMACjBUa84Xq94SM4stmz9QghhGgSPB6A3nzzTe666y7Gjx9P+/btmTVrFt7e3nz88cdn3EdRFCIjI7VHRESEtk1VVaZPn87zzz/PtddeS6dOnZg7dy5Hjx5lwYIFF+ET1Y2ycUC/H/6dYnvDDHJnFH+Fa1C06oQvb4WUPzxdkRBCiEbOowHIZrOxadMmBg8erK3T6XQMHjyYNWvWnHG//Px8YmNjiYmJ4dprr2XHjh3atpSUFDIyMtyOGRAQQGJi4lmPWd91DO1IlE+U6zTYkUZ2Ggxcl8WHtIa8o/DpSFj6D3CUeLoqIYQQjZRHA9Dx48dxOBxuPTgAERERZGRkVLlPmzZt+Pjjj/nuu+/47LPPcDqd9O3bl8OHDwNo+9XkmFarldzcXLdHfdNoJ0Us4xsOdy+HrrcDKvzxOnw8VOYIEkIIUSc8fgqspvr06cOYMWPo0qULAwYM4JtvviEsLIwPPvjgvI/5yiuvEBAQoD1iYmJqseLaU/E0mNVh9XA1dcDsC9fOgBvmgCUAjmyEWZfB1nmerkwIIUQj49EAFBoail6vJzMz0219ZmYmkZGR1TqG0Wika9eu7Nu3D0DbrybHnDRpEjk5Odrj0KFDNf0oF0WnsE5EeEdQUFLA6iOrPV1O3elwHdy7Clr0BVsefHs3fHO3XCYvhBCi1ng0AJlMJrp3787SpUu1dU6nk6VLl9KnT59qHcPhcLBt2zaioqIAiIuLIzIy0u2Yubm5rFu37ozHNJvN+Pv7uz3qI52i03qBFh9c7OFq6lhgDIz7Ea54DhQ9/DkPZvWHQxs8XZkQQohGwOOnwCZOnMhHH33Ep59+ys6dO7nvvvsoKChg/PjxAIwZM4ZJkyZp7f/+97+zaNEiDhw4wObNm7nttts4ePAgd955J+AaK/Poo4/y8ssv8/3337Nt2zbGjBlDdHQ0o0aN8sRHrFVlAWj5oeXYHI38Tuo6PQx4Csb/AoEtIPsgfJzkun+Y0+Hp6oQQQjRgBk8XMHr0aI4dO8aLL75IRkYGXbp0YeHChdog5rS0NHS68px26tQp7rrrLjIyMggKCqJ79+6sXr2a9u3ba22eeuopCgoKuPvuu8nOzqZ///4sXLiw0oSJDVGX8C6EeYVxrOgYa9PXcnnzyz1dUt1rkQj3roQfH4PtX7smTNy/HP72AQQ093R1QgghGiBFVeUulKfLzc0lICCAnJycenk6bOq6qXyx6wuujb+Wl/s3odmTVRW2fum6eaotHyyBcM270P4aT1cmhBCiHqjJ72+PnwITNVd2GmzZoWWUNKW5chQFutwM96yA6K5QnA3zb4cfHgFbgaerE0II0YBIAGqAuoV3I8QSQp4tj3UZ6zxdzsUXEg93LHLdRwwFNs2BDwdC+p8eLkwIIURDIQGoAdLr9AyOdc103eivBjsTgwkGT4YxC8AvCo7vgX9fCWtmgtPp6eqEEELUcxKAGqiym6MuTVtKibMJnQY7XauBrjmD2owAhw1+fRY+vwHyz3wzXSGEEEICUAPVLaIbwZZgcqw5bMho4nPj+ITATf+FEW+CwQL7lsD7fWFvE+0dE0IIcU4SgBoog87AoBaDgCZ8GqwiRYGeE1z3EwvvAAXH4L//B788A/ZGeNsQIYQQF0QCUAOmnQY7uBS70+7hauqJ8HZw1zJIvNe1vO59+OhKOLbbs3UJIYSoVyQANWA9InsQaA7klPUUmzI3ebqc+sNogWGvwi3zwTsUMrfBBwNg48euuYSEEEI0eRKAGjCjzqidBluUusjD1dRDlyTBfasgfhDYi1wzSb/XG9Z9CMU5nq5OCCGEB0kAauDKToMtSVuCQ+6PVZlfJNz6NQz5Jxh94Ngu+OVJeKOdawJFmTtICCGaJAlADVyvqF74m/w5WXySzVmbPV1O/aTTQd8H4fGdMOw1CGsLJQWuCRQ/uAz+fZXrFhslxZ6uVAghxEUiAaiBk9NgNWAJgMS74f61MO4n6PA30Bng8Hr49h54sx0segFOpni6UiGEEHVMAlAjUHZvsCVpS3CqMgvyOSkKtOwPN3wCj/0Fg54H/+ZQdBJWvwPvdIXProfdv4CcVhRCiEZJAlAj0CeqD35GP44XHSc5K9nT5TQsfhFw+ZPwyFa46QtIGAyorskUv7gJ3u4MK16XmaWFEKKRkQDUCBj1Rq5ocQUAiw7KabDzojdA2+Fw29fw8Bbo+zB4BUHOIVj2D3izPXw1HlJXyaX0QgjRCEgAaiTKToMtPrhYToNdqOBWMOQfMHEXXPcBNO8JzhLY8Q3MGQ7v9YH1H0FxrqcrFUIIcZ4kADUSfaL74GP0Iaswi3c2v0NhSaGnS2r4jBbofBPcuQTu+QO6jwOjNxzbCT8/AW+0hR8ehYxtnq5UCCFEDSmqKv35p8vNzSUgIICcnBz8/f09XU61vbnxTT7Z8QkAoV6h3N/lfq5LuA6DzuDhyhqR4hzYOg82/BuOV7i9RvNe0GM8tB4CPqGeq08IIZqwmvz+lgBUhYYagFRV5deDv/L2prc5nH8YgFYBrXis+2MMaD4ARVE8XGEjoqpwcJUrCO38ASreiy2qs2v26fhBEJMIBrPn6hRCiCZEAtAFaqgBqEyJo4R5u+cx689Z5Fhdt3zoEdGDx3s8TsfQjh6urhHKy4TNc+Gv71z3HavI6O265L4sEIVe4roMXwghRK2TAHSBGnoAKpNry2X2ttl89tdn2Jw2AIa1HMZD3R4ixi/Gw9U1UnmZcGA57F/mehScdvm8fzOIv8IVhuIGgk+IB4oUQojGSQLQBWosAahMen46M5Jn8MP+H1BRMegM3NTmJu7pdA+BlkBPl9d4qSpk7igPQwdXg8NaoYEC0V3Ke4ea9wKDyVPVCiFEgycB6AI1tgBUZtfJXby58U3WpK8BwM/ox12d7uKWdrdg1ss4lTpXUuQKQfuXwf7fIGuH+3ajD8RdVh6IQhLkdJkQQtSABKAL1FgDUJnVR1bzxqY32HNqDwBRPlE81PUhRrQagU6RmREumtz08tNlB36DgmPu2wNiKpwuGwDewR4pUwghGgoJQBeosQcgAIfTwY8HfuTdLe+SWZgJQLvgdjzW/TH6RPfxcHVNkNMJmdvLT5elrQGHrUIDBSI7QnRX1yOqC0R0kCvMhBCiAglAF6gpBKAyxfZiPtv5GbO3zSa/JB+Afs368Vi3x2gT3MbD1TVhtsIKp8uWuSZfPJ3OCBHt3UNReHsZRySEaLIkAF2gphSAypwsPsmHf37IvF3zsKt2FBSuTbiWB7o8QKRPpKfLE7lH4fBGSE+Go1vgaLLr7vWn05tcPUNlgSi6K4S3A73xIhcshBAXnwSgC9QUA1CZtNw03t78tnZTVYvewu3tb+eOjnfga/L1cHVCo6qQneYKQxVDUXF25bZ6c/nps7JQFNbWdQNYIYRoRCQAXaCmHIDKbD22lTc2vsGWrC0ABJmDuLfzvdxwyQ0YpTehflJVOJV6WijaCqWTYboxWCDy0vJAFN3FNUmj/GyFEA2YBKALJAHIRVVVlh1axvRN00nNTQUgzCuMvtF96RPdh95RvQnxkon86jVVhZMH3HuJ0reCtYo72esMEBTnCkKhrUufS197BV7kwoUQouYkAF0gCUDuSpwlfLv3W2Ymz+Rksfu4k7bBbekT1Yc+0X3oFtFN5hNqCJzOqkORLe/M+/iEVx2MAmJAJ1MnCCHqhwYXgGbOnMlrr71GRkYGnTt35t1336VXr15Vtv3oo4+YO3cu27dvB6B79+5MnTrVrf24ceP49NNP3fZLSkpi4cKF1aqnLgNQ7oki9m3M4tKBzTGa9bV67LpmdVjZnLmZNelrWHN0DbtO7nLbbtab6R7RXQtElwRdIjdgbShU1TXQ+vgeOL639Ln0dd7RM+9n8HJN2FgWjMJKw1FwPJi8L179QghBAwtA8+bNY8yYMcyaNYvExESmT5/OV199xe7duwkPD6/U/tZbb6Vfv3707dsXi8XCq6++yrfffsuOHTto1qwZ4ApAmZmZfPLJJ9p+ZrOZoKCgatVUVwFIVVX+mLeXvBNFNG8XTOdBDft+XCeKTrA2fS1rjroCUVaR+32vQiwh9Il2haE+UX0I8w7zUKXigljzSkPRacHo5P7T5iqqSIHAmPLeopAECIqFwJYQ0ByMlov5CYQQTUSDCkCJiYn07NmTGTNmAOB0OomJieGhhx7imWeeOef+DoeDoKAgZsyYwZgxYwBXAMrOzmbBggXnVVNd9gDlHCvkr5XpdB8ai8mr8VyFo6oqB3IOsProatYcXcPGzI0U2Yvc2iQEJtAnug99o/vSPaI7XgYvD1UraoXDDtkHKwej47uh6NTZ9/WLgsAWEBhb+tyiNCC1cJ1Wk8HYQojz0GACkM1mw9vbm//973+MGjVKWz927Fiys7P57rvvznmMvLw8wsPD+eqrr7j66qsBVwBasGABJpOJoKAgBg0axMsvv0xISNUDdq1WK1Zr+U0qc3NziYmJuWhjgI4fzic42gedrvGcLrI5bGw9tlULRH+d+AuV8j9qRp2RruFdtR6idsHt5DYcjUnBiQqhaA+c2Oe6bP/UQSgpOPu+ig78ossD0ekhyS9aLuEXQlSpwQSgo0eP0qxZM1avXk2fPuW3X3jqqaf4/fffWbdu3TmPcf/99/Prr7+yY8cOLBZXt/qXX36Jt7c3cXFx7N+/n2effRZfX1/WrFmDXl953M3kyZOZMmVKpfUXIwAdP5zHuu9TCI72odfVcegNjTMEnCo+xbqMdaw5uobVR1eTUZDhtj3IHERiVCKdwjoRHxhPQmACYV5hMoaosVFVKDwJ2amuQFQWirLTXL1J2WlgLz77MXQG8G9Wodco1tVr5B/lCkf+UWD2uygfRwhRvzSZAPSvf/2LadOmsXz5cjp16nTGdgcOHCA+Pp4lS5Zw5ZVXVtruyR6gjJQckhenERUfSKdBzZvEL3xVVUnNTdXGDq3PWE+hvbBSOz+THwmBCVogKnsdYglpEt9Tk6SqkJ9VIRAdrBCQ0iDn0FnGHVVg8isNRFHgH33ac2lQ8g0HXcO6EEEIcXY1CUAe7UcODQ1Fr9eTmZnptj4zM5PIyLPffuH111/nX//6F0uWLDlr+AFo1aoVoaGh7Nu3r8oAZDabMZs9c/l2ZFwA/W+4BC8/o/ZLXVXVRv0LXlEU4gLiiAuI45Z2t1DiLOHPY3+yLn0de07tYX/2ftLy0siz5bEla4s2GWOZQHOgFooqPgdb5G7pDZ6igF+E6xHTs/J2pxPyM07rNToIOYchNx3y0l1zHNny4Hie6/TbGd9LD74R7kHJP7q8F6ns2eRTd59XCOExHg1AJpOJ7t27s3TpUm0MkNPpZOnSpTz44INn3G/atGn885//5Ndff6VHjx7nfJ/Dhw9z4sQJoqKiaqv081ZVuPENcg9fO1YcQVWh/WXR6PWN85RYRUadke4R3eke0V1bZ3VYSc1JZV/2PvZn79eeD+UdItuazabMTWzK3OR2nGBLMPGB8cQHuIejQEvgRf5Eos7odOVBJbZP1W2seZCXAblHSkPR0fJwlHvU9ZyfCarDte1sl/kDmANcgcwnHHxCXT1HPuHgG1b6HA4+Ya5nowzsF6Kh8PhVYPPmzWPs2LF88MEH9OrVi+nTpzN//nx27dpFREQEY8aMoVmzZrzyyisAvPrqq7z44ot8/vnn9OvXTzuOr68vvr6+5OfnM2XKFK6//noiIyPZv38/Tz31FHl5eWzbtq1aPT11dRXYoZOF3P/fzUy97lIubR5Q9XsfL+KP+XtBVek9Kp6QZnL/rYqK7EWk5KS4haJ92fs4kn/kjPuEeoVqwaiZbzOifaOJ8o0i2ieaQHNgo+5tE2fgsENB1mkB6fTndLDl1+y4Jr8KwaiKgFQxRJl8XT1eQoha02DGAJWZMWOGNhFily5deOedd0hMTARg4MCBtGzZkjlz5gDQsmVLDh48WOkYL730EpMnT6aoqIhRo0axZcsWsrOziY6OZsiQIfzjH/8gIiKiWvXUVQB69MstLEg+SqiXnnn39yc+rOpwk3Uwl/yTVlp1lXlzqquwpJCUnJRKPUZHC87+v3svgxeRPpFE+5SHIu3ZJ4ow7zAMOrniqMkqzi3vMcrPgoJjpc9ZkH/M/bk6Y5MqMniVhySfUPAKBu+yR4jr4VX2Ohi8gmR6ACHOocEFoPqmrgJQXnEJz075D7FbVrIo8VrmPDaE6MBzd5mXWB1k7M+hebsg6a2ooYKSAg5kH2Bf9j5SclI4WnCU9Px0jhYc5XjR8XPur1f0RHhHaKEo0ieSaN9oLShF+URhMcikfk2eqkJxzhkCUhYUHK/w+hiUVB70Xy3mgDOEpArrvCpuCwKDqXY/qxD1mASgC1RnM0GXlJD64mQ2bNzLEZ0Xyy+7nk8eG0qwz5n/gVJVlY0/p5KVmkurrmG06xtda/U0dVaHlYyCDI7mH3U9FxzlaP5R0gvSOZp/lMyCTOyq/ZzHCbYEE+XjCkMhXiGEWEJczxVfW0LwNsqtIUQpa757SCo84ZoeoPCEaxJJt+WT555Y8mxMfq4g5BVY+ggCS2CFdWdYNvvJKTrR4EgAukB1ORO0/eRJUl99nQ2b9pKls7DhytHMenQovuaqT7OoqkrK1uPsWZ9Bn+viCQiTX6IXi8Pp4HjRcS0QlfUepReka+uqunz/TLwN3m6hKNQr1C0gVQxNEpaEG6cDirJLg1FpKDo9JBWerLzMBfzzrjOUBqPAc4cmsz9Y/MES4Hpt9peb5AqPkAB0ger6bvCO7Gz2vfIaGzfs5qTewl/Db2X6Q0MwG848J4mtyO5264wSq6PB3Uy1sVFVlVxbrhaOMgsyOVF8ghNFJ8qfS19bHdZzH7ACL4NXpYAU7BVMgCmAALPrEWgO1J79TH4yk7Zw53RCcbYrCBVnu3qRikqfz7Vcwz+vlSmuHqSyQGQJcA9IVS6fts1gkR4oUWMSgC5QXQcgAEduLjtfnsaWDTvJ1VtIu24MU+8bgr4at8MozLWx6n97adkplITu4TIuqJ5TVZWCkgKOFx13C0jHi45rr08WndTW1TQsASgo+Jv9XaHotJBUcX2gOZAAS4D22sfoI39+RGUlRWcPSKevs+a6BowX59RCeCqlN5X3LJl8S3uWfF3BylT6XPbQlsvandZGJrxsMiQAXaCLEYAAHPn5bJ3yKts2/MXyZp2JvP46pl7X8Zy/kPZvzmLXmnQCwrzp87f4Rnv7jKaoLCyVBaWKoSnbmk22NZsca472yLZm1+g03OkMigF/sz8B5gD8TH6uh9EPX5Nv+bPJD1+j+7OfybXN1+grPU/Cnd1aHoasOa5nbTn3zMsV217IqbuqGL0rh6SyZZOPa52p4uvSR1Wvjd5yeq8ekwB0gS5WAAJwFhTwxxc/MX6fN04U7h8Yz1ND255zv0M7TxLa3BcvP7nCo6krcZSQY8shuzjb9WzNJteaq4WlbGs2ubbTlq25FDvOcc+tavIx+lQKSb4mX/xN/tprP6Mf3kZvfIw++Bh98Da4Xpet8zZ4o5f/pQtwnbqz5ZeHI2uea9C4Nbd0/ZmW81wPW175utrqjXKjuIKSyecMIcnHFabKXpeFL5M3GH1czyafyq/lBr+1QgLQBbqYAajMF+vTmPTNNvROBy/1DmXM3/rWaP/Du09hMOqIbFX1BItCnK7YXqwFohxrDnkleeTb8smz5Wmv80tKl23ly7m2XPJt+dicNZz35hwsekulkHTGZYMrPFUMUF4GL7yNrmcvgxcWvUVO7zV1dlt5mCoLSZWWC0pDU37p64rr8yuszwPVWXe16k3uwcgtOJ3ltdG7itc+rlnJy9brTU1mPFWDuReYKHdzrxZk5xVxcMYsijce5XvFzjXXXV6tffNOFrPtt8M4HU4Sr40ntLnMHi3OzWKwYDFYiPCp3gShp7M5bOXhqDQouQWmknzybeWBqdBeSGFJIQUlBRTYCygscS2XTTVQ7Cim2FHMyeKTtfL5FBQsBosWiE4PSF4GLy04nWl7xYdF7/q+zAYzXnovDDqDBKz6zmACQ+kcSRdKVcFeXBqISgOSNd8VkrSglH9aaCp9LiksXa74utC1XXW4ju+wQZHtwqY8OBNFXxquvEpDUoXXZwtQRotrws7qPuuNDSpoSQCqR+7p35LvvvUl52QJWe/O4DeDwhUjLzvnfj4BJmI7hlCYayWkWfmNG9f9cICSIgcdLo8mKNK13l7iwOlQMVnkRy8ujElv0i7dP1+qqmJz2igoKdDCUaG9sFrLZa/LglSRvYhCe6E2iFxFpcheRJG9qLY+shudoisPRXqzK1CeZdnL4HXG9Sa9CYve4nouXe/2MJgxKBK4PEpRSkOBF1BLs/Srqiv4nDEkneW1rcA1WL2kNExV9dpZOo+Z6ijt9cqtnbrPRNGdJSBZXN9dxefWV0G7kXVb01nIb8F6RGc2c+0bz/P5Y/+EHdtJfesdNhgVeg7tf/b99Dra94/G6XS/0WpOVhElxXZ0FW6ompWay5ZFaYS18KPXyFba+mNpeRjNevxCLDKoWlw0iqJov+SDLbXwv3Rc8zcVO4pd4afEFYrKglBZSCrbVnH92bYXO4optrt6qJylp0GcqtMVwC5gEHpN6BSdK0BVCEpuwen05wpByqQ3ac8VX5t17uvMejNGvdFtP5POtV0Gu9cBRQGD2fWgdv78u3GUlAenkqIzhyZtW2Hl1/ZiKCkGe9GZn8uoztLjFVSvPu8QCUCinM5sZvRbzzPn4Zcx7trOX6+9g5cOOg45ewgC0FW4hF5VVfpcF09BttXtbvPFBa7/EZi93e8plLwkDVuRnf43ttYmW8zOLORYWh5BUd6ENverjY8nRJ3T6/T46Fzjhqjlm7OrqkqJs0QLRFa7lSJHEVa71S0klT27ra+wzeqwVlp2e9hdzxXHWTlVZ532aJ2LUWesMkSZdCaMOiMmvQmj3ui2bNK7XldcLttu1Bsr7V9Ve6Ou9Jh6IwadQdtu1BulV+xc9MbyGcDriqq6rvyrFIxKH1qIOv25GFok1l1d1SABqB4ymk3cNv05Pnron/jv3U7ytLex6CHhynOHoDKKouAXbMEv2P0+Va26hBHbIQSHvXwwn9PhxDfIQoHOindAeVg6diiPPeszaNYmyC0Arf1uPwaTno6XN8Pi4wpStmI7qlPF5CX/IInGS1EU7Rezv6nuL5Bwqk5sDlulYFT2KHYUY3PYyp/tpy2XhjCb04bNYdOOpT073dedvl2tcDl6ibOEEmcJlNT5x66RioGpYjjSXutKg1OFcFbl9grtKq4742t91etPX1dxvUFnaHw9aYriOsVltNT6fzjqmgSgesrLy8y4d55j1kP/JDBlN88sTmNGj2IiAy78xpt6ow69sfwvoU6vo8918ZXa+Yd60bxtsNu4IkeJkxOH8wHoPChGW39w+wn2rMugedtgOl9Zvn7fpizM3gaiEgIwGOUyZyFqQqfotMHqF5uqqthV+1kDUsVgVeIscb0uXWd32t2Wy7aXOEsocZSU7+u0YXfY3ZZLHCXu7Svsc7qy7RfrVOSF0it6LQwZdAYMSuWQVPG127JiqHq9znDG41bcXtb+bG31Or22zqgzotfp3dsr7m30On2DDXUSgOoxf28z46c/yz1v/MzmYjO3z17H/Hv6EHSWm6fWpoiW/kS0PO1/uTroMTyOonwbBlP5H3q7zQGKgsW3/NSao8TJ7rXpAG6X56f+eZzDu0/RvG0QLS8N1dbnHi/C5GXA7C29SEJ4mqIoGBVXT4aP0efcO1wEqqriUB1u4cjutGuBye1RVYg6bVvFZbtafhy7065tq/TacY7tpz2fzqE6cDgc5zXje32lU3TlIUlxD1IV11cMTgadgRGtRnBjmxs9VrcEoHouzM/CO48M5f/eX8PerHwmvv0zr/UOInTQAI/Uo9friIir3PXfrm80bRIjcTrLu8wdDicx7YOxFtrdwlLeyWJysgoJa1F+Ws1hd/LHvD0AJN3dUestOrTzJFkHc4mKDyC6dRDg+kfw5NECTF4GfAPNKNW4fYgQouFTFEX75ellqP/nW8p60SoGotMfpwemsvanb690jNPanb7sUB3aPg6nw62Nw+kKkXbVXr7tLMcoe7Y77W6nRcs4VSdO1ek6RVoD3SO619ZXfV4kADUAzYO8+c+EXox7Zwk9l33F6lVO+tttBA+5ytOludHpdW633DFZDHS6IqZSu1Zdwghr4YdPYPl4I1uRHZO3AUeJ0+0qtOzMQjL25+AbVH4KwGF3snbBfqA0LJW+aeq242QcyKFZ6yBi2ruuqFBVlczUXEwWA4HhXm5XxAkhRF2q2Ivm1dAGyJyBU3W6BaayAFUxJGmvKwSuqtrH+sd69LNIAGogWkf4MePuAXx4aDveB7ew9t1P6Gu3Ezh8mKdLqzGfQLNb+AHw8jNx1fgOqKr7pfzN2gThF2whILz8Hw+7zYlPoBl7idNtXFHeiWJOHM4nOMp9zNKmn1MBV1gqC2i712WQtuMELTuF0rqHayJAVVXZuvQQBpOetn0itWPnnSymMNeGT4DZ7Yo6p1N1u/JOCCEaO52i0y4EaOgkADUgXWODueX5e/jk5Y+4/OAmNnz4GT0ddgKuvrrRjJk5/XMER/m4BRoAi4+RgbdWvl9abMcQgqN88Asp7y2ylzgJjPDGbnMPSyXFdmxFdpwO1a3tkd2uWVjb9Y3S1h/ZfYr9m7No2SmUDpc1A1xh6ZdZ29DpFQaNaYfZy/VX6fDuUxzdc4rwlv5u45v2bsxEp1No0TEEo8lVR2GujcJcG16+RrdAaC9xoNfr5NSeEELUIQlADcxll4ST+8QdfPG6wuCDGzHOmU8Xu4OAUdc2mhB0vvxDvfAPde9mtvgY6fd/rSu1vaRXJC06hrjNiK0orrFMJTYHOn35d2n2MRIQ7u02RYCjxAmqitOuYqhwRV3BqWKOpeW5BRpVVdmzPhNUlWZtgrQAdHRvNrvXple6cm7JJztxlDgYeFtbfErf88ieU6QkHycs1o82iZFa2z3rM1CdKrGXhmpTEhTm2sg/VYzF14h/SPn34ShxojMoTf7PiRBCgASgBmlEpyiyH7iVH97TMzR1HcYvvqVzSDC+l1fv3mECTF4GTF7uf/wNRj2tulae4j6uUyhxnULd1umNOq6a0IESq3tYiowPxDvQjK9bAILYDsE47KoWfgCMZj2+QRa3K+dUVdXmaKo4Fqo4v4ScY4X4BrufOkz98zglVgfN2gRDaUfZsbQ8tv9+mIhWAfQY1lJru/yL3RTn2ej3f60JjHBNdpl1MJd9G7MIjvahbZ/yXq+9GzMpsTpoeWko3v6uru7CXBsnjubj5Wtyu99c/ikrqqri7WfSplcou8eyhC0hRH0lAaiBujUxluzC6/npUz0dTxwgy9Sc6zxdVBOiKAomi6HSPdUCwrwICHPvhdLpFDoOaF7pGLEdQ4jtWPk+WkPv6ojd7sRkLg9LUQmB+IVYMHu7v1/spaHYbQ5MXhWDlQ7/UC98AtzP0TtKSoNVhR6rovwSTmUUuO0PcOivkxTl2YiKD9ACUHZmAX8uPURIM1+3ALR5YSp5J4tJvLaVNmFmVmoeG39JJTjKx22OqS2L0yjIttKuXxQh0a5j5B4vYv+WY3j7m9x6tw7vPoW1oISIuABt7JWt2M7J9AKMJj0hzcprKM4vQVVdE3HKrVyEENUhAagBu39gPCcLhvPxH/vRfb8LP38fLvez4SwqwhAWhj4wUP4H3sAoioLeqLiFFABvf5MWRCqqGBjKRLcO0qYMqOjKMe2w250YKwSrsBg/ug1tWSkAxXYMwVpkx8u3/D1NXgZCW/gRcNppRoNJj9HiHjwcDtcpwtPlnywm93gRTnv5tuKCEo7uOYV/mJfb50nbcYJT6QV4Vxh8nneymE0/p+ITaHYbB/bnb4c4lpZHpytjiGnrugIw90QR6747gJe/if4VToPu25RFzrFCWnQIISzGFdhsxXbSdpzEaNET26E8lOafKqbE6sA7wKyN83I6VWxFdnR6RW4qLEQDJn97GzBFUXhueDtOFdr4ZvMRHvh8M18GHiB033bXdqMRfWgIxvBw9KGhGMLC8OnbF52p4Y/eFzV3+gzgcOZgFd8tvNK60OZ+Vd4Tru/1CZXWRcT5c+W49pXWdxzQDFuxA/8KvWQ+gWba9YvGaHEPYeGx/vgEmPH2Lz9FqNfrCIzwxsvPvWZFp6DT69xCmN3mxFZkx2ByP+6pjAKyUnMJb1E+n1Vxfgm716Zj8jK4BaA9GzJJ35tN+8uaaadBi/JsLP9sF3qjjqF3X6q13bb8MEd2n+KSxEhadXGdSrUV2Vn9zT50eh2XjW6t/Yckddtxsg7m0ax1IM3auMKqw+Fk56p0dDqFtn0itSkbTqYXkHu8iIAwL4IiXec5VVXl+KF8dHqFoEhvra2t2I7D7hrwXzHoCiEqkwDUwOl0Cq9e34ncohKW7MxidvIxehU7CS0pwEuvYMnKwbz3IBaDHotJj0/fvtq+2d8uwHYwFUNYmOsRGoYh3PVaZzaf5V2FODu9Xofep/KpqLJf4BX5BJi1wFBRQvfKISwwwrvKQe09R8QB5WOPAAJCvbhs9CWV2sZ2DCGshR+Bkd7aOoNJT/O2wegN7j2mJosBLz+TW5hQnYCiVJoCwWF3usZvVej4cjicFGRbUXTug8/zThRz7GAugRHug9QPbjsOQNs+5T1hGQdySEk+RlyXsPIA5FRZ/8MBAK6a0AFTaQBK/fM4ezdk0qJDCJcOLD/t+tN7f4KqcuW49tpg+QNbjrFrbTox7YLd2i6Z8xd2m5PLRrfWBuGn/XWCXWsyiGzl7za31x/z9mCzOuh1dZx238Gsg7ns33yMoChv2vYuH1e2c3U6JVY78d3CtePmnSwm40AOPgEmt17LrIO5OEqcBEf7aDduthbZyTtRjNGsdzvNnH/KitPhxMvfpI2xc9idWAtdvXRlnxdcM9arKugNiswJJiQANQZGvY4Zt3Rj/Ccb+JGe/AjonA4CbAUEF+cRXJxLkDUPS4GVpS8vIyrQi+hAL0auXUvEyaNYjHrMRh0Wox6LQYdOUdAH+BP1z3+iGFx/RKwHUgAVfVAQeh8fFOlFEvVQxZChN+oqXRUIrp6l03n7m9yuxCvT8fJmcHkzt3W+QWZG3N8J1el+iq99/2gu6RXpFpZMFgO9R8VXOhvYvG0QgRHebvXp9AoJPSJwOlS3KRD8QixExge4tVWdrqsenQ610pgnRadUaxyU06miOlW32dvBFRLKxouVcdhVSort2G3u64sLSrAVud/uoTi/hJNH8zGa3WtI35dNUZ6NmHbBWgDKPV7EnnUZhDTzdQtAO1enk3+ymMRr47UAdCqjgE0/p1YKwclL0sjJKqTH8DhtlvqT6QWs//4AfiFeXH5TeQje8HMqJ4/k03VILNGtA7Xjrvl2Pz6BZgbc3EZru235YU6mF9AmMVK7lU9hro3tK45g9jK4/Xk5tPMkeSeKiWwVQHC0K6SWWB0c2nkSnV5xmxLj+OF8CnKsBFX4+TtKnKQfyEFRoNkl5d9DzrEiCnNt+AaZtYDpdDg5Xno/xrAWftqf+aJ8GyXFDkxeBi30qaqKrdiBTqdgMOlkSMRpJAA1Ehajnv/emcifR3I4cqqIo9lFHM0pfc4uZlN2EScKbGBzsC8rn31Z+ew3tSUsIJKg4lyCC3IJKs4j2JpLoFqCw9/G6i+3El0aljr89B/80tMwG/QY9AoGsxmdjw86X1/0gQGEPfCAVkvR1q04i4pd23180Pn6uEKTt7f8BRSNxunzNLkGxbu30Rt0boO1ywRF+lTqDTMY9VWO6YppG6yNa9KOa9RV2bt1Sa9ILulV+RiDx7cHVXUbsxR7aQjN2gRWCkuXjb4EVPCqcHVis0sCCW3u6zblA0DiNa1wOlS306ihMX50S4rFdNqA/VZdwyixOtxOX3r7m4hpH4xvoPsXFxjujcmix1ThtKjBqMM3yIK3v3vvtMmix+xtRG90/3noDTp0p/XolfXOVfxnyOlwBcHTA21hno38k8XYK4TBEqudYwdz3a7cBFePVcb+HLwDTFoAshba2bnqKAaz3i0AHd51kiO7T9Gub7QWgGxWO1uXpKHoFLcAlLbjBGk7TtC6VwR+wa6fq73EyYYfUwAYdl8n7bOkbD1OSvIxWnUNo13faNfHdaos+XgHUNpTWPrz37shk32bs4jtEEL7/tHa+/3++W4Aeo9qpQXPQ7tOcnDbCSLi/LVJYwHWfLsfh91J92Gx2ljBo3uzOZB8jNAYX7fev3XfH6Ck2EHXpBZa+K0PJAA1IjqdQpeYQLrEBFa5vbjEwdHsItJzijmSXRaOXMtbSl8Xlzgx2634lhRzYnuGtu//7c0lJs+Kn60InepE0SkYdTqMeoUSH39+89tIgJeRQC8jPRbOI+DYUQx6BaPe1cao12HQ6zAHB9HsX69ox81btgzHyZNamNJZLCgWC4rZjM5iwdSihdb29FmihRDVY/aq/E+90aR3m5ahTFW/oKq64hGosoftTOPKKoaAMlUFQaDK3rjQ5n4MuKVNpfW9RraqtC4sxo+h91xaaX3ita1Qne49bIGR3gwa265S23Z9o7B1DXeb/d3L10SnK2PQ693/HYqKD8An0Ox2ak5vVIhuHVQpmAWEeWG3OfEOLP+OdHodoS38Kp1W9QkwExTl43YxgqIoBIR5u/49rNDWYNRh9ja6jXmr2LtXcboOh8OJ0+50O2Wsqir5p4orfQ/WAjs5WYVuE8wC5BwrxFHidJtM1lrkanv6zz/3RBG2QnulnkVPU1S1iks1mrjc3FwCAgLIycnB379yd3ljpaoqpwpLtGDk6kVyD0vH86wYSqx4210PL7sVnepkT1B5UBlycD0RBSe17d72YkwO103y8sw+zO5/O4FeRgK8jIxa/y2R2emugKRTMOgU9DoFvU6Hzmwif9I/8DEb8DLqMf/3Y5TUAxi8LBi9LOi9vNBZzCgmM4rFTMidd5Z3B2/div3UKXRmM4rZ4mpnNqOYTCgmE4bwcK2tBCshRGOlqqoW+sr+nSuxOiixOtAbdVo4VlWVUxmFqE7VbWB9QY6V/FNWvHyNboE362AuqgohzXy0WfbLJmE1exvdwuDxw3k4HSrBUT6VLkqobTX5/S09QEKjKArBPiaCfUx0bBZQZRtVVSmwOcgutJFdWEJOUUn5c5GNnMISsgtjyCqyuW3Lyy+GokKMTjvZha71AHpDM0J9fPEqDUvmEhsmhx2zw0aJzskHn2zQ3vuOHXtokZtZXq9OQa+4QpNqMPKltR1eJj3eJgMD1iwgMiO1NEy52mmvdQrHn/knFrMBi0GPzzf/xbj7L/QmE3qzCUPpQ282ozMZCX34Ye3KuYLVq7EdPOgKUkaj61HhtXf37tr4qJKMDBy5uSgGg+uh10OF1zp/fxSdDMQUQtQdRVFQTuuxMporXyWoKEql2w6Bqxeqql7BM42lq6r3r6qrR+sDCUCiRhRFwddswNdsoHnlqWbOymp3kFNU4gpJWjjqTHZheVjKs9optDkosNkpsjlob3NQaHOt++HSoTiLCtHb7ZidJZgcJZgdJZicdnSqk62Hc7T3shX6EKWPdG0vcbUzOu0YnXZ0qsprn2/R2t686wBtTmVV8WFBryi8XdARk8mIxaRn6LYltM7ch04BvU5xDRjXKa5lRSF5/OMYfHwxG3S0WPEzQTs2adt1iqu9rqz9My9hCQ3GbNTh/PE7Slb+gd5oQGc0ohj0oC8NSwY9oQ88gCHEdXl2wdq1FG7ahGJwtVMMhtJg5Vr2HTQIQ5Drh2NLTcV28KDbsVztXW1NLVqg83ZdDeUsKMBRUICi15eHNZ3O1V6vB50MohRCNB71IgDNnDmT1157jYyMDDp37sy7775Lr169ztj+q6++4oUXXiA1NZXWrVvz6quvMnz4cG27qqq89NJLfPTRR2RnZ9OvXz/ef/99WreufPmsuHjMBj3hfnrC/SznbnwGqqpitTspqhCSCkpDUtnrIpudQlt7CkvXF1gdnLQ5KCxxUGRzYLU76FnioKjEQXGJk7U+SayxFuMotuIoKUGx20vDkgOD08HxQjsUOQD4zRjFzmAzRqcdg9OOwelwa/vlyjTsOtdfqysOnaJDtg6d6sTgdKBXna6H04FBdfDaR+soNLq6iUekbKNnRjrgCplaYCoNS/P1ayj2C8CgU+ixazWX7t+MorgGcyooKJS+VhRWZvlRFByOTlFovX018X+udP0vkArtS9vuvW481ugYDDqFsK1riVy1qMJ2Vw1l++XdMgE14RIMOgXzn5uwLPkFxWBAVxqqdHq99tpyzSjMCQkY9Dqc+/dSsuJ3FL0OfWmIUvQ6QAGdDt8BAzC3cl3Gbjt4kPyVK0Ep7a5XdGWFoCgK3j16YGrZEoCSzEwK160r3a5D0RtA73pWDHrMCQkYo10DPB35BdhSDriCXel21+vS2v380fuWXmLucKDabG4hUAjR+Hg8AM2bN4+JEycya9YsEhMTmT59OklJSezevZvw8MrzgKxevZqbb76ZV155hauvvprPP/+cUaNGsXnzZjp27AjAtGnTeOedd/j000+Ji4vjhRdeICkpib/++guL5fx/+QrPUxTFdbm+UU+QT+1fiq+qKiUOlWK7g+ISB8U2J8V2V3AqLnFQVNKT4hInVrd1TlfbEge32Z1Y7a7t1k5R7Cptay17Ltte4sCrxIHOoWK1O1gS04M/ojujVx3onU7Xc2lw0qlODuc4sOflAnCMcDZHJZaGKkd5uCrdb/XeXApMrlOMHU446KCLQO90YlAdWhgzOJ3oVAf/W3eEY96FACSmH2HQsWJ0qopBdaCcNjzwk592cdDf1cvWK+MvhqccOuP3+N/0UPYGHQSgS9YeRu3/w227opSHsB82FbE/IgG9TqHtsQOM2LHYLYSVhzeFdVtyOBTXAb1OR/PMFPquWoCutFeqrHNKKR0aurvXYA636YZOUQjOSqProi9cx0EBra1LWq8rSO/cF0UBv+PpdFjwSWnb0gPr9drzyZ6XcarHZRj0CpbsEzT75lPQ6dHpda6wpNO5TnHqdJR07o69d3/X2LaCPCzffolOp0On10NZIDToUXQ6DG3aYkzs48pzNisl33+LotO7JnnU6VB0enR6BUWnx9SyJV7du6HXKSgOBwWLF7sG9io6VwjU6VyvdQrG8HAs7dtrf74LVq/Wfgbal4zrWR8UiOWS8qvLCjdvwXXplOs708KgoqDz9dOCK0Dx7j2u6/NLj+mqwVWTzmLWwii4Tg/jcECFNmXHVwwG9IGBWltHfkHpcXWln1Ep740s66EU4jx4fBB0YmIiPXv2ZMaMGQA4nU5iYmJ46KGHeOaZZyq1Hz16NAUFBfz444/aut69e9OlSxdmzZqFqqpER0fz+OOP88QTTwCQk5NDREQEc+bM4aabbjpnTU11ELTwHIdTxVYanIorhqWS8nUlTicOh4pDVXE4VexOFYfTicMJDqezdFnF7lBxqqrbssPpxFG2zlG+zaG6lkucTpxOlRJtuxO7w4lqt+MosaM67BQpRkrQUeJwYiguwlKUj9NhR7U7XL0mDgeq3Q4OB4d8wsjRm3GqEFKUQ8vcdHSqCqilz6CUvt4d1IITXq4xZ6GF2bQ/mYqCilJ6lYvr2fXYERxHuq/raqLwwpN0z9qDojrRqSq6sh620jC4KaIN+wJdE/xF5x9jRMqa0u2ukKlTVS1o/ta8GxsiXVcCxeRlMmH7j5V+RmV+i+nG7827umooOMn9f357xrYrm3ViSYueAAQV5/LIlq/O2HZdZHt+iesDgK+tkCc2fXHGtlvCL+G7+MsAMNttTNrwn9KMcnoYhF1h8Xzf7kpXjFGdPPXHv7XjnP6P/96gGOa3T3KtV+GZNZ+gdzqqrCHFP4q5HYaVvp/Ck+s/w8tupUK+1J6P+kfwWddrtSB737r/4mctKG1TflpVUeCETzCf975RC8m3rfmS4IJstx7Psvco8AngxyHjtbF9V/32BUE5x8qDlU5BKe1FLPH2Yf2ou9HrwKDT0f63b/E/ka59ByoKaukX5zCY2DLqDm3+ptarfsYv85D2iVxtXdWrisKGkeNd61VI2LiMwIw07ViUhf3SorcNuxUMBnSKQkzyKoKOHNDCaNlnVktDf+qw0WA2oygKYdvW439wb3nb0mOrgKroOHzF1ZRYfHCqKoG7tuKbth8nivb5nIoOZ2n7Q136Y/X2xalCwKF9BBxJRQWtvVPR4cTVA5zVriuqXwAGvY6gzDSCjqai14Fe0aHXua4+NpSOsyzu2hNdcAgGnYIlPQ2vfbvR6RRXe52i7aPX6TD1SiQwthn+FvcpBS5UgxkEbbPZ2LRpE5MmTdLW6XQ6Bg8ezJo1a6rcZ82aNUycONFtXVJSEgsWLAAgJSWFjIwMBg8erG0PCAggMTGRNWvWVBmArFYrVqtVW87Nzb2QjyVEjel1Cl4mPV51fIXExeYsDWp2p9MVuJyuHjZHhXX20pB2tuWyUDbKLdSplDgv15ZVSq94UcGpqrRUQaV02XkJKv1wqipOV0PsKthKl7uh0lUtnRxQbcmhwd3A6YTSUKc6XZcNO50OIgxmRhotrra2YLa3Go/T4cTpdKI6HDgdTld7h4Miiz/tvP1xOlUUm5GVuqGoqhMcTq2d6nSAUyXLOxhvkx6HU0XFzPLYHqiqE8WpamFRUVV0qBzxKb+k3Kno2BTRxrVNrdC2NEQe8gohz+qarFBRnewMKJ/1WQuZqKDCYa9grPbyS5VTfcMxqA4tKenKXqgqx7wC0K6yVlWyLAFYHLYqw+tJneX/27v3oKjq/g/g7wPCcomLSrKgiGjeMqHywqCWEzIC+iSU5eXHT/HJIg0dHXOyi4pOPWNlWZOPQzojWI+laZOXydJBUivylqKiKY/6I8wUvMVVUdzz+f2xcGRld2EJWJZ9v2Z22j3nc758v/vhmx++Z/cclFfdu2DiX3CHQbl7X6wKRYDSu8Cl0ntfx66oqobHHdOLLda6IXdwsOCG9vrhG5VQK2+ZjS13v4MNhy5or184+ye6lxebjb3j6obMnN+111POXECvkj/Nxoqi4MuD99qdlP87+t0oNBsLAP/x/107Tf7s2f8i/Np5i7Gr3PJxy8141uIf/3cSg4vPWIz95M5DKNMZT+PG/n4cUZdPWozdUKHX/uiIvnAMT/55DABQe7K37v+Fvv3LG5drft9G/HkcMRd+tdhuxn9VXPA1Xq8o8vIpxP9+wGLsf3JKEJPwJN6Ir38JgtZi1wLo2rVrMBgMCAwMNNkeGBiIM2fMJ7qoqMhsfFFRkba/dpulmPstW7YMS5cubdIYiMgyFxcF7i4K3MHP0dhuHIB7RZ1BjCt7IsYVw9piTkRgUOPuPa/ZrtaJ0VZkFACI1k4x1nVvwaJmFQmAokTXPK+/sgQAaQpQUztBZBQE9/okYlwRqd32stQtUp80rjTUiVNrlltEgH/UxBr3R+HuXdW4imkwGG85UlOQ6lXBv93dteJaHdMLUn0Xd1VjcWm4a4CqCgwGAzwEmOfbSYt1e/R/UXq7CorUFIAKtOeiuOCVwK7aeL3Cn8ftO1U1BVvNG1FbZCrA3G49tffU81Ef4GaF9h4Zx6vWFN7AjJCHoCoKVAG8+nngVkVpnffL+GaoqkAgeCakJ8TVFaoKPND9CVwrewQQ4/V71JplOhcArhDE9QiDuLsbT/l2F9z8qwdcavrrIqpxQQyAiwgmDugHg6cnXBQF/n8q8Lj0oFaIusC4mqrU/Iz/Ce+PSi9fVBtUPPCHCunmWWfl2dhXAwR3VaBfj67Qe/qhWlXxgEcoCjzuGnMlMP4RI4AqxmNveflC18G+f/Dx5CmAN954w2RVqaysDCEh9S/ERUTU2mpPi7hAaTiYbFD/ooqW9bMh1pYv29jSbngLxdrSh/o3OHZkdv2zLCAgAK6uriguNl2GLC4uhl5f/3LuAKDX663G1/7XljZ1Oh18fX1NHkRERNR+2bUAcnd3x6BBg5Cdna1tU1UV2dnZiIqKMntMVFSUSTwAZGVlafFhYWHQ6/UmMWVlZTh48KDFNomIiMi52P0U2Lx585CcnIzBgwdj6NCh+Pjjj1FZWYl//tP4qfqpU6eia9euWLbMeP+oOXPmYOTIkfjwww8xduxYbNy4Eb/++ivWrFkDwLhcPHfuXLzzzjvo3bu39jX44OBgJCYm2muYRERE1IbYvQCaOHEirl69isWLF6OoqAiPPvoodu7cqX2I+cKFC3CpcyGyYcOG4csvv8TChQvx5ptvonfv3ti6dat2DSAAeO2111BZWYmUlBSUlJRgxIgR2LlzJ68BRERERADawHWA2iJeB4iIiMjx2PLvN7+bSkRERE6HBRARERE5HRZARERE5HRYABEREZHTYQFERERETocFEBERETkdFkBERETkdFgAERERkdNhAUREREROx+63wmiLai+OXVZWZueeEBERUWPV/rvdmJtcsAAyo7y8HAAQEhJi554QERGRrcrLy+Hn52c1hvcCM0NVVVy6dAk+Pj5QFKVZ2y4rK0NISAj++OOPdn+fMY61/XKm8XKs7ZczjddZxioiKC8vR3BwsMmN1M3hCpAZLi4u6NatW4v+DF9f33b9S1gXx9p+OdN4Odb2y5nG6wxjbWjlpxY/BE1EREROhwUQEREROR0WQK1Mp9MhLS0NOp3O3l1pcRxr++VM4+VY2y9nGq8zjbWx+CFoIiIicjpcASIiIiKnwwKIiIiInA4LICIiInI6LICIiIjI6bAAagGrVq1Cjx494OHhgcjISBw6dMhq/ObNm9GvXz94eHhg4MCB+O6771qpp023bNkyDBkyBD4+PujSpQsSExORn59v9Zh169ZBURSTh4eHRyv1uOmWLFlSr9/9+vWzeowj5rRWjx496o1XURSkpqaajXekvP744494+umnERwcDEVRsHXrVpP9IoLFixcjKCgInp6eiImJwdmzZxts19Y53xqsjbW6uhoLFizAwIED4e3tjeDgYEydOhWXLl2y2mZT5kJraSi306ZNq9f3uLi4Btt1tNwCMDt/FUXB8uXLLbbZlnPbUlgANbOvvvoK8+bNQ1paGo4ePYqIiAjExsbiypUrZuN/+eUXTJ48GdOnT0dubi4SExORmJiIkydPtnLPbbNv3z6kpqbiwIEDyMrKQnV1NUaPHo3Kykqrx/n6+uLy5cvao7CwsJV6/PcMGDDApN8///yzxVhHzWmtw4cPm4w1KysLAPD8889bPMZR8lpZWYmIiAisWrXK7P73338fn3zyCT799FMcPHgQ3t7eiI2NRVVVlcU2bZ3zrcXaWG/evImjR49i0aJFOHr0KL755hvk5+dj3LhxDbZry1xoTQ3lFgDi4uJM+r5hwwarbTpibgGYjPHy5cvIyMiAoigYP3681Xbbam5bjFCzGjp0qKSmpmqvDQaDBAcHy7Jly8zGT5gwQcaOHWuyLTIyUl5++eUW7Wdzu3LligCQffv2WYzJzMwUPz+/1utUM0lLS5OIiIhGx7eXnNaaM2eO9OrVS1RVNbvfUfMKQLZs2aK9VlVV9Hq9LF++XNtWUlIiOp1ONmzYYLEdW+e8Pdw/VnMOHTokAKSwsNBijK1zwV7MjTc5OVkSEhJsaqe95DYhIUGio6OtxjhKbpsTV4Ca0Z07d3DkyBHExMRo21xcXBATE4P9+/ebPWb//v0m8QAQGxtrMb6tKi0tBQB06tTJalxFRQVCQ0MREhKChIQEnDp1qjW697edPXsWwcHB6NmzJ5KSknDhwgWLse0lp4Dxd3r9+vV44YUXrN4Y2FHzWldBQQGKiopMcufn54fIyEiLuWvKnG+rSktLoSgK/P39rcbZMhfamr1796JLly7o27cvZs6cievXr1uMbS+5LS4uxo4dOzB9+vQGYx05t03BAqgZXbt2DQaDAYGBgSbbAwMDUVRUZPaYoqIim+LbIlVVMXfuXAwfPhyPPPKIxbi+ffsiIyMD27Ztw/r166GqKoYNG4aLFy+2Ym9tFxkZiXXr1mHnzp1IT09HQUEBnnjiCZSXl5uNbw85rbV161aUlJRg2rRpFmMcNa/3q82PLblrypxvi6qqqrBgwQJMnjzZ6o0ybZ0LbUlcXBw+//xzZGdn47333sO+ffsQHx8Pg8FgNr695Pazzz6Dj48Pnn32WatxjpzbpuLd4OlvS01NxcmTJxs8XxwVFYWoqCjt9bBhw9C/f3+sXr0ab7/9dkt3s8ni4+O15+Hh4YiMjERoaCg2bdrUqL+qHNnatWsRHx+P4OBgizGOmlcyqq6uxoQJEyAiSE9PtxrryHNh0qRJ2vOBAwciPDwcvXr1wt69ezFq1Cg79qxlZWRkICkpqcEvJjhybpuKK0DNKCAgAK6uriguLjbZXlxcDL1eb/YYvV5vU3xbM2vWLHz77bfYs2cPunXrZtOxbm5ueOyxx3Du3LkW6l3L8Pf3R58+fSz229FzWquwsBC7d+/Giy++aNNxjprX2vzYkrumzPm2pLb4KSwsRFZWltXVH3MamgttWc+ePREQEGCx746eWwD46aefkJ+fb/McBhw7t43FAqgZubu7Y9CgQcjOzta2qaqK7Oxsk7+Q64qKijKJB4CsrCyL8W2FiGDWrFnYsmULfvjhB4SFhdnchsFgQF5eHoKCglqghy2noqIC58+ft9hvR83p/TIzM9GlSxeMHTvWpuMcNa9hYWHQ6/UmuSsrK8PBgwct5q4pc76tqC1+zp49i927d6Nz5842t9HQXGjLLl68iOvXr1vsuyPnttbatWsxaNAgRERE2HysI+e20ez9Kez2ZuPGjaLT6WTdunXy22+/SUpKivj7+0tRUZGIiEyZMkVef/11LT4nJ0c6dOggH3zwgZw+fVrS0tLEzc1N8vLy7DWERpk5c6b4+fnJ3r175fLly9rj5s2bWsz9Y126dKns2rVLzp8/L0eOHJFJkyaJh4eHnDp1yh5DaLRXX31V9u7dKwUFBZKTkyMxMTESEBAgV65cEZH2k9O6DAaDdO/eXRYsWFBvnyPntby8XHJzcyU3N1cAyIoVKyQ3N1f75tO7774r/v7+sm3bNjlx4oQkJCRIWFiY3Lp1S2sjOjpaVq5cqb1uaM7bi7Wx3rlzR8aNGyfdunWTY8eOmczh27dva23cP9aG5oI9WRtveXm5zJ8/X/bv3y8FBQWye/duefzxx6V3795SVVWltdEeclurtLRUvLy8JD093WwbjpTblsICqAWsXLlSunfvLu7u7jJ06FA5cOCAtm/kyJGSnJxsEr9p0ybp06ePuLu7y4ABA2THjh2t3GPbATD7yMzM1GLuH+vcuXO19yUwMFDGjBkjR48ebf3O22jixIkSFBQk7u7u0rVrV5k4caKcO3dO299eclrXrl27BIDk5+fX2+fIed2zZ4/Z39va8aiqKosWLZLAwEDR6XQyatSoeu9BaGiopKWlmWyzNuftxdpYCwoKLM7hPXv2aG3cP9aG5oI9WRvvzZs3ZfTo0fLggw+Km5ubhIaGyksvvVSvkGkPua21evVq8fT0lJKSErNtOFJuW4oiItKiS0xEREREbQw/A0REREROhwUQEREROR0WQEREROR0WAARERGR02EBRERERE6HBRARERE5HRZARERE5HRYABERNYKiKNi6dau9u0FEzYQFEBG1edOmTYOiKPUecXFx9u4aETmoDvbuABFRY8TFxSEzM9Nkm06ns1NviMjRcQWIiByCTqeDXq83eXTs2BGA8fRUeno64uPj4enpiZ49e+Lrr782OT4vLw/R0dHw9PRE586dkZKSgoqKCpOYjIwMDBgwADqdDkFBQZg1a5bJ/mvXruGZZ56Bl5cXevfuje3bt7fsoImoxbAAIqJ2YdGiRRg/fjyOHz+OpKQkTJo0CadPnwYAVFZWIjY2Fh07dsThw4exefNm7N6926TASU9PR2pqKlJSUpCXl4ft27fjoYceMvkZS5cuxYQJE3DixAmMGTMGSUlJuHHjRquOk4iaib3vxkpE1JDk5GRxdXUVb29vk8e//vUvEREBIDNmzDA5JjIyUmbOnCkiImvWrJGOHTtKRUWFtn/Hjh3i4uKi3RE8ODhY3nrrLYt9ACALFy7UXldUVAgA+f7775ttnETUevgZICJyCE899RTS09NNtnXq1El7HhUVZbIvKioKx44dAwCcPn0aERER8Pb21vYPHz4cqqoiPz8fiqLg0qVLGDVqlNU+hIeHa8+9vb3h6+uLK1euNHVIRGRHLICIyCF4e3vXOyXVXDw9PRsV5+bmZvJaURSoqtoSXSKiFsbPABFRu3DgwIF6r/v37w8A6N+/P44fP47Kykptf05ODlxcXNC3b1/4+PigR48eyM7ObtU+E5H9cAWIiBzC7du3UVRUZLKtQ4cOCAgIAABs3rwZgwcPxogRI/DFF1/g0KFDWLt2LQAgKSkJaWlpSE5OxpIlS3D16lXMnj0bU6ZMQWBgIABgyZIlmDFjBrp06YL4+HiUl5cjJycHs2fPbt2BElGrYAFERA5h586dCAoKMtnWt29fnDlzBoDxG1obN27EK6+8gqCgIGzYsAEPP/wwAMDLywu7du3CnDlzMGTIEHh5eWH8+PFYsWKF1lZycjKqqqrw0UcfYf78+QgICMBzzz3XegMkolaliIjYuxNERH+HoijYsmULEhMT7d0VInIQ/AwQEREROR0WQEREROR0+BkgInJ4PJNPRLbiChARERE5HRZARERE5HRYABEREZHTYQFERERETocFEBERETkdFkBERETkdFgAERERkdNhAUREREROhwUQEREROZ3/B1lvWE9bdivlAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ],
      "source": [
        "plt.plot(Adam_model, label='Adam')\n",
        "plt.plot(SGD_model,label='SGD')\n",
        "plt.plot(Adagrad_model,label='Adagrad')\n",
        "plt.plot(RMSprop_model,linestyle='--',label='RMSprop',alpha=0.7)\n",
        "plt.plot(LBFGS_loss,linestyle=':',label='L-BFGS',alpha=0.7)\n",
        "\n",
        "plt.xlabel('Epoch')\n",
        "plt.ylabel('Training Loss')\n",
        "plt.title('Training Loss over Epoches')\n",
        "plt.legend()\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Evaluate these methods on test sets to compare their accuracy"
      ],
      "metadata": {
        "id": "C2XigOirWSyA"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "id": "YDpEh0QP6wXx"
      },
      "outputs": [],
      "source": [
        "def evaluate(model):\n",
        "  model.eval()\n",
        "  correct = 0\n",
        "\n",
        "  with torch.no_grad():\n",
        "    for test_imgs, test_labels in test_loader:\n",
        "        test_imgs, test_labels = test_imgs.to(device), test_labels.to(device)\n",
        "        test_imgs = test_imgs.float().to(device)\n",
        "        output = model(test_imgs)\n",
        "        #calculate accuracy\n",
        "        predicted = torch.max(output,1)[1]\n",
        "        correct += (predicted == test_labels).sum()\n",
        "\n",
        "    accuracy = correct / (len(test_loader.dataset))\n",
        "    print(\"Accuracy: {:.3f}%\".format(accuracy * 100))\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Iw10_BWs6x4A",
        "outputId": "0a4bddf2-da8f-43df-935a-8a838bf121d2"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy: 99.230%\n"
          ]
        }
      ],
      "source": [
        "evaluate(cnn_adam)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cCkylVI7hZx1",
        "outputId": "a1b5714c-cd4b-48b5-b9be-f696a62352e4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy: 96.090%\n"
          ]
        }
      ],
      "source": [
        "evaluate(cnn_sgd)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "id": "eTjrh_TpjNLu",
        "outputId": "2b0c81d7-525f-4cbb-8c04-ecb74c8022d2",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy: 98.980%\n"
          ]
        }
      ],
      "source": [
        "evaluate(cnn_RMSprop)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "evaluate(cnn_Adagrad)"
      ],
      "metadata": {
        "id": "ICOQznZRf1oG",
        "outputId": "e42f10c6-ee74-481b-ddde-a22a24ac95c4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy: 96.850%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "evaluate(cnn_LBFGS)"
      ],
      "metadata": {
        "id": "KDuFMsTk15K5",
        "outputId": "60702400-81a5-4dbc-d33a-0166d6d21880",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy: 98.600%\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "As shown above, Adam acheive the best accuracy, and RMSprop and L-BFGS also provide competitive results."
      ],
      "metadata": {
        "id": "EegUj77IWZ0-"
      }
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "gpuType": "T4",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.11.5"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}