First_try_complete_batch_sigmoid_relu_momentum.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "colab": {
      "name": "First_try_complete_batch_sigmoid_relu_momentum.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.7"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "thermal-humanity"
      },
      "source": [
        "import torch\n",
        "import math\n",
        "#from dlc_practical_prologue import *\n",
        "#torch.set_grad_enabled(False)"
      ],
      "id": "thermal-humanity",
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RAhIEBMckCI_"
      },
      "source": [
        "#INTERESTING FACT: WHEN COMPUTING THE BATCH GRADIENT PYTORCH DOES NOT TAKE THE SUM OF THE GRADIENTA OF SINGLE DATA POINTS\n",
        "#BUT THE MEAN. I LEART AT THE EXPENSE OF ABOUT 1H LOL "
      ],
      "id": "RAhIEBMckCI_",
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "found-annex"
      },
      "source": [
        "class Parameter():\n",
        "    def __init__(self):\n",
        "        self.name = ''\n",
        "        self.data = None\n",
        "        self.grad = None"
      ],
      "id": "found-annex",
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "therapeutic-current"
      },
      "source": [
        "class Module(object):\n",
        "    def forward (self, *input):\n",
        "        raise NotImplementedError\n",
        "        \n",
        "    def backward ( self , * gradwrtoutput ) :\n",
        "        raise NotImplementedError\n",
        "    \n",
        "    def get_parameters( self ) :\n",
        "        return []   "
      ],
      "id": "therapeutic-current",
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "imperial-class",
        "scrolled": true
      },
      "source": [
        "class Losses(object):        \n",
        "    def forward():\n",
        "        return NotImplementedError\n",
        "    def backward():\n",
        "        NotImplementedError"
      ],
      "id": "imperial-class",
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gxGfS8ZckCJC"
      },
      "source": [
        "class Optimizer(object):\n",
        "    def zero_grad(self):\n",
        "        for parameter in self.param : \n",
        "            parameter.grad = 0\n",
        "            \n",
        "    def step(self):\n",
        "        raise NotImplementedError"
      ],
      "id": "gxGfS8ZckCJC",
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "comfortable-calgary"
      },
      "source": [
        "class SGD(Optimizer):\n",
        "\n",
        "    def __init__(self,lr,max_iter, parameters, momentum = True ,beta= 0.9) :   \n",
        "        super().__init__()\n",
        "        self.eta = lr   # learning rate is fixed\n",
        "        self.maxStep = max_iter\n",
        "        self.param = parameters\n",
        "        self.number_step = 0\n",
        "        # Initialize momentum to zero\n",
        "        self.v = self.param.copy()\n",
        "        self.v = []\n",
        "        for i in range(len(self.param)):\n",
        "            size_param = self.param[i].data.size()\n",
        "            v_init= torch.zeros(size_param)\n",
        "            (self.v).append(v_init)\n",
        "        self.beta = beta\n",
        "        self.momentum = momentum\n",
        "\n",
        "    def step(self): \n",
        "\n",
        "        if self.number_step <=self.maxStep:\n",
        "            for parameter, V in zip(self.param, self.v):\n",
        "                \n",
        "                if self.momentum :    # SGD + Momentum\n",
        "                    V = self.beta * V + self.eta * parameter.grad   \n",
        "                    parameter.data = parameter.data - V\n",
        "                else :                # SGD \n",
        "                    parameter.data = parameter.data - self.eta * parameter.grad\n",
        "           \n",
        "            self.number_step = self.number_step + 1\n",
        "        return self.param"
      ],
      "id": "comfortable-calgary",
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "asian-evanescence"
      },
      "source": [
        "class Linear(Module):\n",
        "    \n",
        "    def __init__(self, input_dim, out_dim, bias = True):\n",
        "        super().__init__()\n",
        "        std = 1/math.sqrt(input_dim)\n",
        "        self.weight = Parameter()\n",
        "        self.parameters = []\n",
        "        \n",
        "        self.weight.data = torch.rand(out_dim, input_dim)\n",
        "        self.weight.data = 2*std*self.weight.data - std\n",
        "        self.weight.name = 'weight'\n",
        "        self.parameters += [self.weight]\n",
        "        \n",
        "        self.with_bias = bias\n",
        "        if bias :\n",
        "            self.bias = Parameter()\n",
        "            self.bias.data = torch.rand(out_dim)\n",
        "            self.bias.data = 2*std*self.bias.data - std\n",
        "            self.bias.data = self.bias.data.unsqueeze(0)\n",
        "            self.bias.name = 'bias'\n",
        "            self.parameters +=[self.bias]\n",
        "            \n",
        "        self.x = None\n",
        "              \n",
        "    def forward(self, x):\n",
        "        self.x = x\n",
        "        self.batch_size = x.shape[0]\n",
        "        return self.x.mm(self.weight.data.T) + self.bias.data\n",
        "        \n",
        "    def backward(self, prev_grad):\n",
        "        \n",
        "        prev_grad = prev_grad.view(self.batch_size, -1, 1)\n",
        "        #print(prev_grad.shape)\n",
        "        if self.x is None:\n",
        "            raise CallForwardFirst\n",
        "        \n",
        "        if self.weight.grad is None:\n",
        "            self.weight.grad = torch.zeros_like(self.weight.data)\n",
        "        \n",
        "        grad_on_batch = prev_grad.view(self.batch_size, -1, 1)*self.x.view(self.batch_size, 1, -1)\n",
        "        self.weight.grad += grad_on_batch.mean(0)\n",
        "        \n",
        "        if self.with_bias:\n",
        "            if self.bias.grad is None:\n",
        "                self.bias.grad = torch.zeros_like(self.bias.data)\n",
        "            grad_on_batch = prev_grad.view(self.batch_size, -1)\n",
        "            self.bias.grad += grad_on_batch.mean(0)\n",
        "        \n",
        "        #if the output has dimension one, squeezing creates problems\n",
        "        if prev_grad.shape[1]>1:\n",
        "            prev_grad = prev_grad.squeeze()\n",
        "        next_grad = prev_grad@self.weight.data\n",
        "        return next_grad.squeeze()\n",
        "    \n",
        "    def get_parameters(self):\n",
        "        return self.parameters    "
      ],
      "id": "asian-evanescence",
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "written-benjamin",
        "scrolled": true
      },
      "source": [
        "class Tanh(Module):\n",
        "    def __init__(self):\n",
        "        self.x = None\n",
        "    \n",
        "    def forward (self, x):\n",
        "        self.x = x\n",
        "        return torch.tanh(x)\n",
        "        \n",
        "    def backward ( self, prev_grad) :\n",
        "        if self.x is None:\n",
        "            raise CallForwardFirst\n",
        "            \n",
        "        def d(x):\n",
        "            return 4 * (x.exp() + x.mul(-1).exp()).pow(-2)\n",
        "        \n",
        "        return d(self.x)*prev_grad\n",
        "            "
      ],
      "id": "written-benjamin",
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0xO9pboykCJE"
      },
      "source": [
        "class ReLu(Module):\n",
        "    def __init__(self):\n",
        "        self.x = None\n",
        "    \n",
        "    def forward (self, x):\n",
        "        self.x = x\n",
        "        x[x<0]=0\n",
        "        return x\n",
        "        \n",
        "    def backward ( self, prev_grad) :\n",
        "        if self.x is None:\n",
        "            raise CallForwardFirst\n",
        "            \n",
        "        def d(x):\n",
        "            x[x<0]=0\n",
        "            x[x>0]=1\n",
        "            return x\n",
        "        \n",
        "        return d(self.x)*prev_grad\n",
        "            "
      ],
      "id": "0xO9pboykCJE",
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Iti6kv4BkCJF"
      },
      "source": [
        "#Doesn't work and we have to output 1 unit so it's useless\n",
        "class Softmax(Module):\n",
        "    def __init__(self):\n",
        "        self.x = None\n",
        "    \n",
        "    def forward (self, x):\n",
        "        self.x = x\n",
        "        return torch.softmax(x,1)\n",
        "        \n",
        "    def backward ( self, prev_grad) :\n",
        "        if self.x is None:\n",
        "            raise CallForwardFirst\n",
        "            \n",
        "        def d(x):\n",
        "            return -x.mm(x.transpose(0,1))+torch.diag(x)\n",
        "        \n",
        "        return d(self.x)*prev_grad"
      ],
      "id": "Iti6kv4BkCJF",
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "udjGPzn7kCJG"
      },
      "source": [
        "#Doesn't work and we have to output 1 unit so it's useless\n",
        "class Sigmoid(Module):\n",
        "    def __init__(self):\n",
        "        self.x = None\n",
        "    \n",
        "    def forward (self, x):\n",
        "        self.x = x\n",
        "        return torch.sigmoid(x)\n",
        "        \n",
        "    def backward ( self, prev_grad) :\n",
        "        if self.x is None:\n",
        "            raise CallForwardFirst\n",
        "            \n",
        "        def d(x):\n",
        "            y=torch.sigmoid(x)\n",
        "            return y*(1-y)\n",
        "        \n",
        "        return d(self.x)*prev_grad"
      ],
      "id": "udjGPzn7kCJG",
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "happy-review"
      },
      "source": [
        "class MSE(Losses):\n",
        "    # Attention! Works well only when the vectors provided are of the form [batch_size, vector dimension]\n",
        "    # Otherwise it doesn know what dimesion to pick for the mean computation\n",
        "    # I'll fix this later\n",
        "    def __init__(self):\n",
        "        self.x = None\n",
        "    def forward(self, x, t):\n",
        "        self.x = x\n",
        "        self.t = t\n",
        "        return (x - t).pow(2).mean()\n",
        "    \n",
        "    def backward(self):\n",
        "        if self.x == None or self.t == None:\n",
        "            raise CallForwardFirst\n",
        "        return 2 * (self.x - self.t)/self.x.shape[1]"
      ],
      "id": "happy-review",
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "active-skirt"
      },
      "source": [
        "class Sequential(object):\n",
        "    def __init__(self, modules):\n",
        "        super().__init__()\n",
        "        self.modules=modules\n",
        "        self.parameters = []\n",
        "        for m in self.modules:\n",
        "            param = m.get_parameters()\n",
        "            if param:\n",
        "                self.parameters += param\n",
        "        \n",
        "    def forward(self,x):\n",
        "        for m in self.modules:\n",
        "            x=m.forward(x)\n",
        "        return x\n",
        "    \n",
        "    def backward(self, loss_grad):\n",
        "        x = loss_grad\n",
        "        for m in reversed(self.modules):\n",
        "            #print(m)\n",
        "            x = m.backward(x)\n",
        "            \n",
        "    def get_parameters(self):\n",
        "        return self.parameters\n",
        "\n",
        "    def set_parameters(self , params):\n",
        "        self.parameters = params\n",
        "        "
      ],
      "id": "active-skirt",
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1g29-IZvkCJI"
      },
      "source": [
        ""
      ],
      "id": "1g29-IZvkCJI",
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "legal-buying",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "outputId": "86b21547-2fd6-4043-9d28-8bb3eda0e8ba"
      },
      "source": [
        "def generate_disc_set(nb):\n",
        "    input_ = torch.empty(nb, 2).uniform_(0, 1)\n",
        "    target = (input_-0.5).pow(2).sum(1).sub(1 / (math.pi*2)).sign().add(1).div(2).long()\n",
        "    return input_, target\n",
        "input_data, output_data=generate_disc_set(1000)\n",
        "output_data=1-output_data\n",
        "input_data-=input_data.mean(0)\n",
        "input_data/=input_data.std(0)\n",
        "import matplotlib.pyplot as plt\n",
        "plt.plot(input_data[:,0][output_data==1],input_data[:,1][output_data==1],'bo')\n",
        "plt.plot(input_data[:,0][output_data!=1],input_data[:,1][output_data!=1],'go')"
      ],
      "id": "legal-buying",
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7fefc70d0d50>]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD4CAYAAADvsV2wAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO29f3Bdx3Um+B08ADJAWrT54E2cOHh0xppMnDBKbJYrjmenXGZqY0Fry+ImW0kBDELbgyWwqaJmNqU4xpQpKovMSJsqm5kEojmKNAjxKhNtirZlkxpvLI8rsx47CZ2NBHuyjmQvwcibjUUwpgQTFkCg94/7Grjvvj7dp/v2fe8R735Vr0i8H3379u0+ffqc75xDSimUKFGiRIndj75Od6BEiRIlSrQHpcAvUaJEiR5BKfBLlChRokdQCvwSJUqU6BGUAr9EiRIlegT9ne6ADSMjI+rAgQOd7kaJEiVK3DL4yle+clUp9TrTZ10t8A8cOIBLly51uhslSpQoccuAiJa5z0qTTokSJUr0CEqBX6JEiRI9glLglyhRokSPoBT4JUqUKNEjKAV+iRIlSvQISoEfgPpSHQc+dgB9p/pw4GMHUF+qd7pLJUoAKOdmCTt6SuDHWAz1pTqmPj2F5evLUFBYvr6MqU9P5V5Y5UJtRSfH5FZ8HkXNzRK7B9TN6ZEPHTqkYvHw9WK4sXFj+73hgWGcfc9ZjB8cF7dz4GMHsHy9leZaHari9F2nMfv0LK5cv4LRfaOYOzwnajtW33YTOjkmt+rz4OZmbV8Nl++73P4OlegIiOgrSqlDps96RsOffXq2aQEDwI2NG5h9etarnSvXrxjfX1lbwbFPHgvSriR9i6lxptsaeXgEIw+PdJ0mG+t53WrXTsP3mXNzk3u/hAy34mmPQ88I/FiLYXTfKPvZxtZG099SIeHqW8yj+syFGRw9f3S7rZW1FaysrQS3W9Ri6KTw6gbByT3zmQsz7Hhzc9M2Z28VdEpJKdpM1u7NpGcEfqzFMHd4zuv7EiFh61t9qY7JT0xG0TjrS3WcuXQGCrwZ78bGDZx46oRoEpoWw7FPHouyGIsQXtLF1Q2CkztlPHLpEVb4zB2ew/DAcNNvhgeGvedstyE7z7JKSqw5Z4LttOeaT5LP2+1z6RmBH2sxjB8cR3WoKv6+REhwfRu7YwxTn57Cpto0/s5X45x9etYq7DVW1lZEk9C0GDa2NlpODDatlENs4SVdXPWlOlbXV1t+327BKX226Y1//OA4zr7nLGr7aiAQavtqwX6HbjJjmOZZGqY5V/RJU1+Hm0+S+XbiqRNtNx32jMDPLobqUBVD/UM4ev6o94Q+fdfpFmE0WBnEQN9A03tSIcEt1IvPXbROdF+NM9QkwU1Ck4PQ9NusVvr+T73fOd4xhRcg95NMfXoKK2srTd+rDlVF144pJH2ebfq5jh8cx+X7LmPr5BYu33c5WNh3mu2THkvJPEvjxsYNTH5iUtRf1zPjngOBrPPJNd/qS/WWeaZRpOlwV7J06kt1K1smBgvDdA0AQSwdDn2n+liNPCbDSAICYevkVtN7/Q/2s6cPF6pDVVy9/6r4+65n6oJtLKtDVXbxAUCFKthSW9brmuYUgaCgUNtX8+5vfamOifMTou/GZuF0mu1jGssQuNaIRA6YvjPQN9Dir9PQ64Sbb/pz21rMO849xdKRaCfc7jtxfkKsmZk0KR/tSqINctpFhSpB2q7JTAIAewb2bGvRnLnK1JdQYQ/AKmCziKFx2jRmV1821WaQeUsv+JD+Sk2HRZiaOEHko3nmOe24TDhSuMwjklOf6aR5+223s23uH9oPwO0Hso1lkabDXSfwuYeYPuLZBrsdx1epAOPs2Av3LgCA94IyTd7FI4tY/fDq9iZlMldxQqW2r+Z760EIoUlmBc7YHWMt90Ug775w13UJwxDbrOlZ9Pf1b/e7QhVM3jkZNTagvlRnx0ULM0kbrvlt2xBsY6nNsdWh6vb/ByuD7PdtbUnZWFlF7traNbbNl155CfWlutMHxW0I1aFqobEeu07gcw9xU21uOxD7yH7bRTtOJJsSwNuxAQRrvK5TiI/t3DSpB/oGthdjbV8Newb2GPvh4/j2pUmaBM7CMwuYvHOy6b4kDmzpdSU2d1/brMnvpM1EQDKnF55ZiM5K4cbl5fWXc8WVTH5icptWaYtZ4caytq+GrZNbuHr/VVy9/+r2/x+75zFUqGL8TR/1edvnXc/SRc2efXrWuY64DeH0Xaet186LKAKfiB4jom8T0VeZz99JRNeJ6K8ar4/EuK4JtodxY+MGzlw6IzJFFOk4sW1KE+cnMPLwyPbkzApoALlompKjttQ0ZZrUj7/v8e3FePm+y/j4ez7e4swe6Bvwmti+C5MTOBefu9h0X6EnFNN1OXOZpL82pJ/F3sG9wbEeUtjm/frmeq64Em0aW1lbsd6HL0Nr/OA4Fu5dMI6/zRxnY8fZ1ojrWS9fX8aBjx0AAHYdxSYlSBFLw//3AN7t+M5/Vkr9ZOP1YKTrtsD1MKRaXZGca1fbK2sr25MzG3By7JPHgmmaJs03u8GkvysxGaUF0tzhOcw+Pdv0m/GD43j8fY+3bAo+E9tXAEhPBHOH56zmABO466YXMNBqLopha29HQJhrbuaJK3FBtx0iDLO/MWn8Evv85J2TWHhmwXp61r/jThWAzDQcg1Hli2gsHSI6AOAzSqkfN3z2TgC/ppT6733azMPSmfzEZLBTsei8KVIWwp6BPbixcUO8Sbm8+zZmQPqeQ1hMReef8WHp+LBM6kt1nHjqhMiJLGXbZNvUeZaA5PSxfH0ZFapgU216MXjawZ5xsYMk1wpl2cS8DxdLhgM3xiamluQ+O5HHqFtYOm8nomeI6Cki+jHuS0Q0RUSXiOjSiy++GHQh7ojHOaOqQ9W2Hq0kGgIAfHfju1525rE7xqyf27QzHw5xFjGjgTn4aEM+J4Lxg+O4ev9VqJMK6qRizTx64UqEfZbLv3ZzDV+88sXt0xWww3Dy8b/kDUaTmvM4/wqBguJKXPMciM82CrXPu8xR6eeVPdX5tNcptEvg/yWAmlLqTgD/FsAnuS8qpc4qpQ4ppQ697nWvC76g6bh2/NBxo5MRQIvmWFSkoW736PmjeM2rXhOlTY2Lz120fi6d7Lbowux4aAEXKxpYt5ln7PPYR/MKVW6zPPuVs6wmKN0Y89yXD7XVxA4iEI4fOi5WhNIbtEn5GqwMNjn3YypZXLT0YGUQq+urQVToNLLRzTZ/ULflMWqLScfw3csADimlrJE3MdMja6RNA/uH9uPl9Zexvrm+/fnwwPC2HS+veSJrhhi7Y6yl3ZjQx1XO/OE6gmotVhKgpcdDmyg4+B5pizYNSfsQGlRnC/CywWVqyAtfc1DeQLei27NdxzTH9w7uxSs3X2lyFpvmldQclX5enFmwUym1bSaddtnwvx/A3yulFBG9DcAfI9H4rRcvQuCnYbPXmTRWH+Fli7wsCtpWbJqwaTuya3JKJ31tXw1Xrl/JFQ2cFQSr66tGe3qoLTSGoPHZhHznlEbRtt5Qm3anEPrcYqzp9LX7qM/6O26t6PXWifoJhdvwiegPAXwJwI8Q0QtE9AEiOk5Exxtf+XkAXyWiZwD8DoBfdAn7dsBmr/P5vgm2yMtQEAjTh6atHHYuSlEzfwDg6v1XsXhkkc0rBKDJdMBBn5JM6KM+kbDPmhli5heJlRNGkhdFm6BW11eNOZWm3jplZY+trq8WGuznysjaLYnSAD82WRYx1rTLHJU28dnWm86omb6vTo9zFIGvlPolpdTrlVIDSqk3KKV+Xyl1Ril1pvH57yqlfkwpdadS6qeVUv8lxnXzwiasTAhNaOVC2mnMRQ5Wh6o4d+Qc5u+eZyP9rq1dEztm9aQ+d+Qc1m6utWQbBHY4xCH2yde+6rVO7cYnhJ67lm0RFV305sr1Ky3CaWVtBUTUYp+ev3u+ycGXnWNpKq7PPUrhyshqy/pYhJCauTCD/gf7QacI/Q/2Y+bCzPZnNiF67JPH2D7Ul+rs2uUcx+kNT/eFTlETrdjmN5FG7cdSPvJiVyZPk2Lk4RGjRrlnYA8UVMsRfvLOSVx87mIuamDWrMPZEW3HWZs9FrBnscwe4SW2XZtJ4+j5o8GmAqm9mzOfuEwtscwYIeMdSpHN/i6mT8M0rzgfjKaLmhKH3X7b7bi2di3YRDZzYQaPXHqk5f3pQ9OYv3veOS9MifdsZkgC4V1vfBe+9MKXjGua86tJxlni78ozT0LQLbTMrgOnKd/YuBEUkJEGp1EdP3TcybJwURBtTBJX4FlWU5YE89i0nDzFQmz5RCRMFDaE/w9m0dcH9K3GK3rDjXdoMJT0dzHLLZrmla0f0noHvlrqx7/yceP7Z7+SpA2RBCZmYTstKih86YUvtaTWcKUgl4yzJML6yvUrXVFFDehxgW8TVtnFYZoYtgnBCcn5u+dzR9fZBLD+zGTnT9se9TGW06SyY8NtQqYJT6Dt8HKbMLDlE5GMEWuv3XMFSgGbn50DNuIUvYm54dXr8s0opqAwmWds/Zdcw3fzqS/VsaXMpyttZw+JgJYkr8um1rBteNJ2JVz80X2jXVFFDehxge/DuQ5ZeCYhGcsmajsF6GCitGM2nXht5OERTJyfsEbdSoWiKaWANC1w3nwi7GK53nh/aRx48iwqq/mD6nw2PNv41evA1FRjM1p3/y6WoOBsyKYsorof0mv4khlc/Rw/ON7i+E7DpMyEJq+Lkf9Iz43FI4teJ+9OlJ/saYHvI3BiLLx2O25MiddMFZ3SyBsEkz0xpLMkmjY4TpDW68CBA0BfX/Jv3TBExuP0+jDwdGoRLY1j87cvY/TxLVz5l5cx+55xY1uh8N20ZmeBGzeSfuHTZ4Hv1ABFqKyafxdLUNgSynH9l5grgLhkBr0evrvxXfY7psR7ocnr1m6usd/3HWfJybvdydKy6GmnrQ9iOM/aWUnIx0GnEcLJ9s2bIuLmN7TgG6kmh4eBs2eB8czPmjjTq6OJ5rzU/CUiID3Nubbagb6+5r5oEAFbzNDHiCUIdWDPXJixFr6PtQbSqO2rWb+jTpr7osdp+fqyiBwBAHSKpx0vHlkUpdJoR0CZD0qnbQTE2KHb5bjhThKuhRZiT/StTiSx+W5rwenf3Ujez6KJM/2TlzH8Dbuwt7XFQXLakGKUGWLufcDtxJf0L/SEevG5i6ywD1kDUicnF2tii0HR46ROKpw7ck60Vjm6ZoUq4rxJsU7s7eDplwLfA3nSmdo4wrEdN9zx3ZbEKtSeGLJZOR1lzMfc+xrj44nmXqslgr5WM2vTkrY09GljeTlpa3k5+TtU6M/NJSeMNIaHk/dDMDMDHD3q7p9J0A7SMFY/NWfdKLhnRaAg0oHUyXn6rtPo7+tver+/r19cR0G6VqfeOsW+7xLAMRlU7TL37nqB3w3RbbYEY0U4bmzRhibtqjpUDbYn2jYrW7CLtc0ALVhjfBy4fDkxj1y+nAj90LYAv9OGBKZNKdS8VK8DZ87ITjDZE2q1vwb15FmsfGHculEUwS6RODmB1uy2ISUpXZi/ex7Th6a352qFKpg+NI13jL7DKIBnLsxsy5MYtX81Ym4eNuxqgd8t0W02s0fsmqSAvUScqabt1fuvBveBcyouHll0hqWzbUbUgk1tAcDqqkxLDz1t2Mws2U2JE/YuU83srN8JJq317v13l7HxleYLmzaKucNzxoplMZQUm5l09unZlqpYunxgbMzfPY+bH7kJdVLh5kduYv7ueVYAn7l0ZluecAjZDNtl7t3VAl+ya7bjBGB7aLFrkgJ2Zkces5QJRTATYmrBuq1qxvS7sgIcOwaMjDhs3wGnjRhmoHo96V+6jYmJpL+6Hdum4zrB+GxkRGT9Ow+4+chpzy4/VCxwa9YVGR56Ym8XT39Xs3RczASOeZNOoaDz7UhDyUPYMe1i6bSbPVCvJxrjlSuJAJqb6wwzBkgE+rJDVpjYOz6MIde1arVEo5dgZCTZlGz9nJ01X4cIOHfOPtbSPraTWZZG/4P9RhNohSq4+ZGbhV1XQ8ImSoNAudZZzBQaPcvSce2akmPbytpKUyi5LWufT3BLGkWEV+fV5POefEK03JhsmCwkTlqj7TvgtMFtLK4NJw1O2Kf7aTJXEQHHj7s3Vu63Y5miaUWaGmxzjMtuGVq21BdcBLkJtX213CfmdvH0d7XAdwWthB7buMyGruCWUCdmEbAtNl/fh0lQ+zo7Y7NhspA6aY22b6HNXaPCkKG490Nw5YrZXLV/P/COd7h/Pz4OTE4mQl5DKWBhoXnMizI1uOaYrdRkO2ASwKaKeQRylha1Ib0OZ5+exdzhuUKLmu9qgZ+lgFWosm3Dt+URkcDkQbdpQ+MHzXV2OxFe7VpsPowBTlBz2iynacdmw2TBOW+zkG4MNmwySij3vglZn0MW6X6upYJFV1bkG+XFi26Gjw+d07Txc4qFa451QyqC7Cl5/u55TN452aTpK6hgP5xpHR49f7QpVXRs7GqBD6ApRDxbONpkavGhfmUFPJdfX78f69iW19ziWmw+x3hOUHPaLCdQfZyIIaafrGmmWgUGM/m5QplA2f5wwpqjiJpw+nRr/zTS/QzdKOt12aYspXPOzLRu/Mc+Wsf7P2GmNroojd2SiiALUyBaKH2SK5J05tKZwpiEu85pG5LzO7TubDYvN5df35S/OxQxnDsuZ7aPo45LFQD4pTSQOhFDnKgcYjiVTf0ZGEjufX2nVPJ2HwH5NXX/lpeTDXRzMxmP9G+CUjUY+pyGzbnMPSfdvybcdwB4jbsmRNO1hc5gGynBh7Bgqjttq3kRs1SkLe9/Hqd4W2raFgFfgc8JQ054cw8pW+j8eze/Z0zmNNA3gA++5YPbE4R7eDHrhsZgTbja8KrheoBniqSnlnYmzs+b+8QJ8snJxPSgBeTqqtmh6cOAiQnu/qtVYO/eZsEOxNusXNcPEdqS/tg2+Bac7ANILl+kiottfgIQz11JHqh0uza2XYiAtjGB8siMnmHpcKYKzkzDmWDStrur91/F6odXjTk8NrY2RIEYfdRnNb/4mGhisCZc9lGf4zTH9sgKBaUSwc3BxIaZnEyciGkzAcdekaZK4BDKEOKue+1aq6O3CD9FSJCabay0sOfGgzPJGU141/0cItJTqs0k6eN/kuSBurFxAyeeOmHNRSX1LWTX+dgdY6xsKorIEauI+WNE9G0i+irzORHR7xDR80T0LBG9JcZ1swhh3UiFLVcdS1Kib1NtsmwXX0ZMDNaERKBLaZ0x89dk2TAXL/Jmhyz2m/duEfIwhHwCs0Kjdm0IoY1yfa7VdoQ9Nx7cBvPOdzYzfgAkaarXBZ7yBrJFvznYlB4fhUiqJK2srbAbg9S3YFrnC88s4F1vfFeL0C/SOR1Lw//3AN5t+fwuAHc0XlMAWgtaRoDvrqjplRJhG2vHzWobvjk0YrEXYkbcxs5fo5FXa5dConlzGq+Php0nR1CMVA0arj7bxoM7iX3pS4aNPpPzH9+pYc/AHrZf0tQnNqWH+0xBtSh0ede0TwI5bp0/f+35psye1aEqhvqHcPT80UIi/6MIfKXUnwIwq8AJ7gHwByrBlwG8hoheH+PaaXDCkEupqmmaaXDC1icQw4W0ZsFpGdzx0UY1TSd26lSiOEAmBCUmFB+t/Zpt9lkgYavYNF6Xhp2+z9XVxKGbhoQZFDtGwdVn10nE6yS2NA587DJwagu1T1wWmVBC6shKajpnNxRpgRcOo/tGWQtB9n0bK0krX+eOnMPazbXcNYNtaJcN/wcB/G3q7xca70UFZ6o4fddp4wThovZMQlgaiCFBWrPgtAwCWUsDmqimj1x6pOOJ4gCZEIwdZBXCn9f9cLXpOgFwGnb2PldWdiihrs0hvQly1z9xwv+eNWynAt+TiOQkpje20FKEabjyN03eOckGOaY3FFM73AnEZHYZu2OsxUIwcX4Ce39rL375/C83vc8hPR7tyJjZdU5bIpoioktEdOnFF1/0/r3JVMFNEC5qTz+E7C4NwBmI0XI/Dvvc3OE54+8VlPVBSx1ORWQXlMAmUKTOS6nWPjycpATwdbqa+pFuU2veobZ3U/vr6wl7x7U5pDdB7jorK/agp1CYTmiDg8kJxdS+zZGb3dgkWvX+of3OkypbGnOpjoVnFqwpGNIbSrYdbk0pqBb5cfG5i8bvf3fju9iCm2GTlQXtyJgZjZZJRAcAfEYp9eOGzz4O4AtKqT9s/P11AO9USv2drc2iSxzmoXely6lxyCZi4zjBXJk1GzXLxuGVttEpSLnjUsrj2FjC5vGlO9oohouLO78NoT7a2jdx5G3XAHizk+blx6Z7pmMU9u8HXnoJ2Ngwty+NjdiOK7i9jsrPzWJzb2spQmwOJPb+/p0gBp84E2kJxTSFUrKWjTEowjXItZeVBbES1XUDLfNJAL/cYOv8NIDrLmHfDrjycXPHq7TH3QaJsAf4/CC246/U4RTqmCoykZnUZMD5Ak6fdtuQJXRHF1vF1Q+X7T0Wg8dFsSyC7pk+oe3d2yzsdftHjyZZPY8eBYaGeFMV0HyC0YXl8YDC8H88h2p/DQCBrteA793eJOwBv5OqSxvOatWStcyRIkLXFufsbUc6iVi0zD8E8CUAP0JELxDRB4joOBEdb3zlIoBvAngewL8DUFyyCE9wR0Pb8Upax/X3/6/fF9nUuQc9dscYe7SVTILQyVJ0IjOpAJVSDkNNLrH7oaE3y+XlVqpiCINnfJxP1zA6WgzdU9KO9kvof9fWkrTMJqYQZz777pfHsfZbl1H9t1tQH70MDJvteFKzhk0ImyiUrrVcoQofgxLo9OX62I50Ersq0jYmbMcrW1StC9zxzBTinU3xkD3acqkc9HVCc3OHmjB8EDNXvqu/tmuF9MPVXta8oQPRsmkRsm3azCK2z7m8+LGely0yV3pNcYQuk46h2l/D1VlDwxn4ph5xmWVcJtGZCzN45JKcZT5YGcRj9zxWaE6gbjDp3DLQjtrl68uswzUPf5fTVLInDZNDKHu05dhHi0cW/Yusp0w4vpkuQ+DLHbeB09THxhKTw8QEf1rx7Yfr9GPSZLWwt7XvOkXYPvd1svpCmmkU4OeImEVlCtZaHwY+JzupSrTkNBmjj+wi0LXWLz5nDh/fM7DHSAfvtILdUxq+K6mSSTvQTqW0xizJwcFB6oCRJmnacThdQWV1FJufnUPtpXEvjdmVTEujWgWuxskBFx1ZrdvkyE0jVPt1nSZsmqxNw88LHydraPuTk+4Uz6YcQvqE8v73NyeUy/5uba3xvA7WgcOzwL4rSXqGp+dAXx1nk8F53YfH2pU4i23rdHTfaEeqhfVM8jQbJEc9zoxTHapi7+DeJnPLE197YtucsmdgD9Y311uKLmcRg21gmix5M0hKj+zdLPCzcN2TLZukDS7mjeS6tiRyMVCUSc6lGLiyhB49yo/duXPJ/7lNJZp5illXFapgU21u/ys1iYaYfotmzpUmHciCGjhzy8raSpPz9ZFLjzTZzhUUPviWD1qvn3X+uHL4+HjsOZbG5KTsSC811YRGs3YCrnsKLXTiYt64zB9KAWfOxGU9ZVGUE9dUUyDNzLn99lYNXgeITU3xJx+lkrbHx5NTWQgjSgpujW+pLaiTCjc/chPqpBKbRG3rtF2FyX3QEwK/vlRnaVfL15e3BW7og9BlDG3l17bUVpOwd+Xw8fHYcwt5c1PGspEKvxjVoNoFW1/zCBAXsyctFDkoFa+SlwlcSooYzy/t87h6NXlp/wenEKys2M2F6bEKSQbng9hC2LZOu6FqVxa7XuBr4WqDDoleWVvBQN+A9bscrly/wkbNAmEh1NIEZ5KFbONlS5xypgLXHIrk8EvB3VO1mk+ASASSFoo2oe+jbfuMZ70OvPxy6/sDA/G0ZA4hG8rAQKtzOaZDPwsuJ5a0Lq3pZM6t026s2rXrBb6UMw8Aq+urIEoy1ukHxCVey2J03yjGD47j+KHjznQKPiHUksUuZVFwQsYkxA4fdhe4NqFoDr8UpntaXEw0Uo4WKRWqUoE0N2dIGdyAVDj6jufsrNkxevvtfk78kA2bO/1wMQREyUtz+ds1V7LrU1qX1jeVORA3K20M7DqnbZaJ44qGNSHtGPWpiiMtsSZ1yPo4Y9Msjb6+/I6vkZGwylLt4PDHhm9qAB/O/sxMYrOXlnrMwnc8Q0oeppGXAJBlCwE7SeOyYzA01N7qZa617GLPxEp9UDR6hqVjo1X6gKM+ZuteLl9f9vbqc/00MXhChWeMRTsxYf7MJTjyCpyiYBPWknHOM6Z5gsx8xzPvhhtrw5YEoNlYO0XMFVeeHRd7JmY92yLRMywdrgp89gg3WBm02uqzDpzssWz+7nljemJpOmKpbc/FtuCO3nkdXzaHossUwX3e19cZW3697g6+krBa8uSryWOT9k1VzPlZpP6XWAwfSQBanoIwIXClZ3A5bruRdeOLXSXwbSUO0xVlXj34amxsbRgdrFIv+omnThgdryeekiUpl9j2bAvCZNudmEiEW17Hl21xuxx/nD9hc7P9tnw9RiazQVpYSwRPLEHoax/3TdzG1Q221RNOwzUW0v5LxssWIV2E05+rYQ3I1n03sm68oZTq2tdb3/pW5YPaR2sKD6DlVftoTSml1OKzi2p4brjps4EHB1T1oaqiB0jVPlpTi88uOq+z+Oyi8Tr6JWlDgsVFpYaHlUpEevIaHk7er9Wa3zd9JxRc29WqvN+VirmNWi28X76wjRGgFNFOf7lxdrXlcz+S62S/r6+rx7NW2/m+/pxo530i+72a2k//3tZHV//T7Umff7YP09N+YyQe+2cX1cCDA8b1uve39ras2cVnF1Xto7UWucC97/pdOwHgkmJkaseFuu3lK/BNAn14bnh70F0bghRcO6HtWe/JsCiV4he2VBBx7erP8i46H8FTFHzGyDYe+vO8Y+KzaUiEq+nzalUuaG2C3TQWtv6b2vNRRFxKTF5FwbZm0zJCKbMcoQdITX9m2noNl/xpF2wCf1c5bQE7QyaW0yVvhr0Y4Fg0232wOL4kDsi82Sw551+lktA7i8gnI+0DEMY8WTFZaOoAACAASURBVF5O+r+5ufOvKT8ON3YxCqIAyTVXV83PvyknjeVeQ5yztjxBejxM729t2eeQNJdTrRY+H11rNs204Zy7BMK5I+dYYka3sHh6xmkL2G3jsZwuoc4dE1wpFkJhc3xJHJB5g184J2E7bfkxgq+aCndgR6jpf7MOYBtvPkZBFH1NbrO/di1/DQHOTs/1n4hPqra15Z5DtlKT6Wvkie2w2e+BZv8fx+RxlR1tR4nCvNh1Aj+NrDAdu2MsitPFVvjAp72QQA4NW14bV+qAogtmAHYnYd5qTD4YGtr5f7VqD74yQSKMdN4irdlzm6mPAzZPrh/JZs21v38/v2GZ+p/l14fchytxn+kasedQWknjCqADduF9K7B4dq3ANwnTR//y0SZmTnWoGhTqnKZVAjsTxDd0Ok+VelvhaJf2GpMOx2mDrs0j5ubC9SvL0Flbs38/5D409MnFVkvAhy7rk4NewydHELf5APyGZeq/TdhL+1Ph5av1GsvLiWmTY/PoZ0o/UcfKDd7+mVXSbAXQbcL7lmDxcMb9bnj5Om3TcDlWtUNl+jPTHfOq0wPE9M3t2czjRIzhgHS142LI+DhMffskubbUSe1qS/oKcTq6rl+t5hu3vCwfpfj+VSqtY8z11TZutmvYnMLbz/TgosKHh1kZUDlVaVnznOygB8gpH9IsnepDVW8GYAygV1g6afDCtPUhxvKq+1Ky2E3pvpqqVt0LOI+gjCFkQ1kbErZJSH8kTBHTNfKyT3yuFYKYY+SCL/1UwiYyMYek489dg3vp32y3eR+zxh5IKNmmNRrK0nG10S7GTk8KfImGH5NWaXvANk4v/auM9vHh4UQrYRY1J6hjaslSuLRBF49cqTj8dldbLuEgvY80V5zjmWdfWU03FKY+FPG8QzYX25y0CepKJflNtarU4KD9mouLsvHWz2z7vZO84ld9iA8sycunj0UBD4FN4EehZRLRuwGcBlAB8KhS6t9kPv8VAP8bgG813vpdpdSjrnbzVLzKU4YwhFbJFRSvDlWxdnONzZtDP1FvKeeGpR2jriSfy+Rkazm/vCXtJIiRd0VCVZRSRMWFsjPXCLkPKZWwiLwwpoRsMStpxSowb6OWmtDXl4wVVwpS0l6tljDEHtF1xZmi6ECx9OlO5t0plJZJRBUAvwfgLgBvBvBLRPRmw1f/SCn1k42XU9jnRTZfTXWoisHKYHPfBbnrJagv1Y3CHkiqZdkcs7WXxoGPXQZObSX/LjXPckk+l7Nnw/O85IFv2L8JklB+aXpgX6ezrUqV6z6yDkzO8cj1KTQFcb3eKuyB5O9YlbQ4lo9vn30d81tbO+Me4sgeHgbe9KaUsAcSBUrFWec+4Np20UOLRgyWztsAPK+U+qZSah3AfwBwT4R2cyPNyb96/1V84Kc+sM2oqVAF73rju6J41SWsmiyWG/Qu1ySW5HPhONBFM2FiVCdyCVufpGVcW9PT8ipVoffxmtcAg836BLtp5KkZMDvLn2KUKm6TD+lzCOvLpqi4SixOTgKf/3zmR0vjwJ8fbxH6dHMYY7f5s2ekcTNzh+eMCRpfXn85WqxNEDhbj/QF4OeRmHH030eRmGzS3/kVAH8H4FkAfwzghyztTQG4BODS6OhoNLsWZ2OPwdKxOYirD1XNzuJ/Udu2UU5Pm+2Rg4Mye3c35K3JA5v/wZc10i4fh8k+PTCQ2KNd18jjt3CliygqdUVIn0Md3qH3YPXhHFxMHLgnKfn34KK349vXEcut/aLt+LDY8NvFw/80gANKqZ8A8CcAFrgvKqXOKqUOKaUOve51r4vWAY7z/sil5Px37sg5a0Ua287OHd+qQ1Wcvus06GZGvVwfhvrc3LYmwwUpvfrVzVomp8FOTcUr/NyO8oTZawB8oJBvzABnjohdNs908tjYAL7zHfdvfQLfsmPF1avVKCy1MNPn5eWkbyMjrZx4SX1fE0LjQaz2/aVW06mv2dM3bubamjk6spORtzEE/rcA/FDq7zdgxzkLAFBKrSilXmn8+SiAt0a4rhdsg+yKcHVFxHIBF6fvOo3xg+NQnzoLfKeWHCu/UwM+fRZYGt9eRNxiykbTcqaH+fk4hZ/bUZ7Q9xpcdKc0v3tRyFM4nhPa2fdNY/XSS62mI408xdldsAlhpZIAN1OpQr3RcqUesxgcbK1x64IepxD4mD19Uyd0ZeQtp/pLXwD6AXwTwBsBDAJ4BsCPZb7z+tT/7wXwZUnbeWiZWUhomtxRS0KxstG4XMfhmNTEPGhHP0KuMT3dasooiocuhZQCarovLqNlNv20LU21i+4aGyHmmTSPnjM7poPHqtXELGZ7zibTnO1ZEO1QWPPObV+qZae4+Ciahw9gDMDfAPgGgNnGew8CeG/j//8awNcam8F/AvBPJO3GEviLzy6qvb+1VxSEZQJno+e+nwVno59uxHGEBtbEtksXmdJYsjA5dMuGmIZUAJruSzrO3ZBiOo30fJMIfCJ5AJ5SYQFYrmfgCvCbnpfz7UMEeCfy4xcu8It6xRD4podUhIafvWb6IVffuchOZElwkvEaBURfFiVYJcIxvajzhvu3C+m+cn00jZ10nLtxo9OQnHBqNXnqBaXczzkvaaEleG3eT4AvPrvY5IitPlTtSIETF2wCf9cmT9MwOVpMGB4YxtgdY0bHrCspUtqhO/LwCI598liTvX/lZ6aAg63GSG3rTKfetfGQm+4rR41VDjF49Sa4Mk7qa3D2fc7mnbUrZx2cMzPFOqC1ffrcOWCglYGHgQHz2EnHuajnAbid867PXX3Q/eRs5FtbrXPc5aC3+U1spRKJgP7+pAQokASoAcAjfyN3wmo/XjreZu2mJRufEEWlR2fB7QTd8ArR8LPatTSdwvRnpq27vS09gugEcV8tWDMxoSitt4gUDTYTgKSiUrXqPs3krbiUB7Z+Z+F7oivieYRW08pem/NFADvmSp9Tiuu6rpxH2dQTTpMbk3bBZKotIlVCUTZ+9ErFK1M6BQIZQ5zT0GmOQ6rVcFVuWqAooYQ1MDzMa72ScPwYaQ3aBVv1q3Q1pKNHk6VowvR0Ql/lwv2lYfxFjI+0kpWk0lg74KpGpqt7ZZEdO1t6CX1fgN8929I6SMevXk+CsLiAxG0waRdMaz40VYKtAl9RFbJ6puKVyXzjEvbaNBNarUbKqa0OjLbQJjl+soSHzB33x8YSPjRR8hoZaU91KRvGxsy0vCyF0cYxX1hI7pnj0UvpdUVEH9uKiaTNIidOhJvhYsZH2Ewjrpz+aYyPJ4LVBFsOfdsGZ4uXkLSlNwWnsAeStAvrskj7EIolR+eeuTBjVRSL5OnvKoFvG6jqUHU7p47+f7pgSShnVsKpHR4Yxun3zrVMZJeN1rbITZN/chJ49NHmoh8rK8D73985oV+vJ8LadZDUgpBLM+ESjNJgnSICk0zPcXAw4cyn/RFcWcLlZbvPIXZ8hG0MbtyQ5wXSz5aD3iBiBr3Z2tKavSuh3TaWxpOYmEaMjK2AUUhxEy5Q68ylM1arQKE8fc7W0w0vXxu+zWbvsrWF2tOmPzNtvN6euT0yqpdHalmXDdrGnOgUs8MnZbGm8dk+59BJG76+fvo52uzb0perCEvoM5WOlWvsXM+2nXPOFKvhO84u+FIspTU5Sht+AyE2/InzE8bP0rY2zq5ms7dxKMoOF2Kjt6UHLiJNrwQ+KYv1vYX6J7L237Ex4IkndjTrahU4fbo99nLfVM0carXkfkxt5XmmNju3Tk/sSpFsu8d2+ibqdbv/Jw2dPtnmD4oFsX9P921fTSRzXLDZ8PtztdxlGD84jhNPnTCmKtbHpKxjV9vV9O99B7uoSvUhhcZHR3n7a1E5Vlzg+pQtTJ2mZnKmD1c6hfHx1iN+2uSwsrITgl+0IOLuu1oF9u6V54nXQsn0/TzPVN+/yQmqBaBrjLh+Seoqx4QtgyjQGcc4kJiBQkgkRWJX2fAB4PRdp622tjyFw7OoL9XRR+YhzGuHCyk0Pjdn5oMPDhaXY8UFzk9x/Hir8w1IBNDqqrktLskchyJiFaTg7vv06eSUIk0opjXQIvj42umqbfaVSvK3VDBy/VpYaK9wdTniTfckcYLn5chna3LU9tVw/NDxFvmk4crpFQWcracbXqGRtiZbm37PN62C7Roc/z6GHS5PuoW0/VhSGzcmTD4JKZfcZRP2jTHodISu7b59fQ6d4OOH3mO7y2265o00ZiO9ViQ+vdC0CS5ZlDd9MnrFhs9BUu7Q1+bO2ecqVMHCvQu57XBAonWcONEZG3QI8vLMXXZvXw59t8cqmHwO7bAtaxQxPp2INZiZyVS5MiB9T7aYDd3X2RftvjmTTEmXLpWgqDKIPcPD5+BKrxBS5Yqz0W+qLcy+ZzxaOP9aKnp7ZSVxThHZ225HTnvTtUyUOIkJRbfhssP6mjCKTE2gkWessxTD+fm4OftdCPETudBOM5oee5ewB5rvyXZ/N24kSpbLNxfDNNyJ9Mk9IfBtDlQb99YG7qHQ9dFofGnT4tFCkWu7HTntuWuFlFpMt8GhWg3TEGOULrShnWMdG/V6skmZkMcZXMQmYoJk3qSRvifX/a2sAPv77cI4BlkjhNufFz0h8DnhrI9nmpLp46AxPSy6mVSySiOPduNaJKa226lhuZKiadgWmK2NWg1YXASuXg0X0rErXaXRSadwHtiiUfOegKSJ7vJCOveA1nty1ZEGAHzOLoxjaOcmp26I8ukFzrjfDa+Y+fBdidFCgq5aErUdNKdBLqRGJ9N2Ox2VkkAXmxMwNMiqW9Bpp7AJEoepT8pin2ssLrYWMAFaazPHgKS2L8CPQZbcYHwdXFSVX6spGJyysROfxcybj17Oh69hy3ZZOVWJ4i3vRERkJ3Oo2wSHi6HhurdORQb7MEy6LV+9lHUjzV7qew2frKF5YVOGfJhpkohoTmnJypTpz0wHs3aymwc9QGr6M9PyAUmhFPiKp2raUht7UzULKEqSXkiSMn9F9MHWN1eaXU542hZsp8oX+o5d0c/bt3xh3sIqEqFpu4btxJOHqslRP32KztjallQuc7WZR+PnKJr0AAVp+j0v8LmHka5eE4sPWyQHWdp23j64fp/+vFpNXr65gGxaZuz+ShGisce6tsvEINlIpCYmqZDzOR3o++c2jxgxJdnfcv2WmtRMm2tom3ny5dty7oTIIJvA7wkevm9OC8CfU7tbwOU41zEAgIxn7eJ4x+KAx+R9S/Pax4Ytr3warrGR1h0YH9+JAXCxXLLXtD23uTnzsxgaMqfLsN2Pa0x0pLItr7/t+XPzJqSvQD5OvU0+hXDyC+fhE9G7iejrRPQ8EX3I8PltRPRHjc//jIgOxLiuFL55bSpU6UlhD/DsB52HRprT3UXPs+XzT5elc8UcxGTKhKSzCEGWu28aUxNcrC2OfZKtO1Cv77CXXCkeste0xTZwNNhr1/zvx8XCWV7mU3DovP42eiw3b/T9pCFhLuVh7cwdngPBUDBC+Hsf5Bb4RFQB8HsA7gLwZgC/RERvznztAwD+QSn1JgAfBfBQ3uv6gBu06lDVSL2KFSkbgnYGTZngCkrhEptlf8cJSaWS+wLM+fwXFppr/AJ2fntM3ne7ArWy3H1uTLPQY8fNiazANeW1z26GLopi9jm6YhtMNNiQjdT1/Ijs4+ba9Ln2r10Li91wcepnLsyg/8F+0ClC/4P9mLkws/298YPjOH7oeIvQL4STz9l6pC8Abwfw2dTfvwHgNzLf+SyAtzf+3w/gKpCYk2yvdtAyY9KhcvdT6AQs0k/gk7/eZud22YkHBlr7HZJfPQ9TJiQPjO/YF5kn3wUfm77NVp4XprlAtFP31oTQeSi1uxfBsOJkCVczQ9vo09/TjMHKqUp3snQA/DyAR1N/HwXwu5nvfBXAG1J/fwPACNPeFIBLAC6Njo5636yNfhkq2Nu1KUgmYdEsHJeg9nHA2ah6uq00JNxqSX91f3yTl7nGkfvN9DTPSx8czC+4QgWTr1DLjhd3X6bvc857DVOBEhcDKrSgiWSMJAnUYoGjfaeVz+nPTEfj9d9SAj/98tXwi6gCX1RleRMkGlk7uN8ujc9Xy7UtyjRCKyhl+1utJgLGJtBDxpH7DSfI8mrzLqqj5DmGKgcSyq1NMcheJ2S884ydpoK67tH0jEynTwnSWTC1kLdlxZS8YrN0YjhtvwXgh1J/v6HxnvE7RNQPYB8AoeVSDldCo5D81j5JkvLa3yW2zhCbdb0uK2yu+3/0aFKkY3rabMvUdtpz55LfHT3aWoM1PQ5S2OzJNlv6F7/Y7BhcWQHOnLE7c0PGkfss0WFaryO1zduul8eRnCeXkMsZ7nKqZm3oIePNOZSrVXdqBKXc98l9vrGRONJ9kC5YDgCbKnFA+bIDs4hd0DyGwP8LAHcQ0RuJaBDALwJ4MvOdJwFMNv7/8wA+39iJosKW0IirIK+FPrcZSJMkuRJpSTYDidPQVwDU60kR82xh82PHks/Sm8HERHP/FxaSa3MFo033OzPT+j6HvXub/04LKGDH6WgTVPV6Ityzs4mbXVrAhAhSH7aOxGlMlAgvMhM0ohQ/Cc0l5BLQkvtLfye0oA9XRCY9T0yQFpjhNmXpZq3lxsT5CWtG3lB0HUtHKXUTwK8iccz+NYAnlFJfI6IHiei9ja/9PoAqET0P4F8CaKFuxoCNGmXT1G2bgZRuxWlEJ07IsypKNDJfATA7C6yvt76vtZhjx/jJbWM6cPd79qw8qdWNG+YxuHw5GaebN5N/bYLKVd4uCy1gQgTp3BwvnE3XqVb5z2u15IS0tmbuf5bqmG5raEjWhzxwCWjJ5pf+Tsh429aDnieLi21gVmWUwZkLMzjwsQOgU4Sj54+KtXiuOh6HIlg6UXj4SqmLSql/rJT6R0qpucZ7H1FKPdn4//eUUr+glHqTUuptSqlvxrhuFjZqlE1Tt20G0hSmnMazssJz1ycmeG1fKeCFF3a+MzOzY24ZGtrRDF3HdJsmtrKSCH4bfI/iXIpkE7a2WjcUX7OY7f6ywjktCELMHePjss1FX+f0aTM1cmBgp0i4aXM01YTN1kUoOg3z2Jh9/Fx0zqzQDTUvucyHeVNgc5uyft+kDD5y6ZFtIW8KtjK2N1TFq/pfJetUA0XEAu2q9Mi2dKM2Td22GdjaTO/8ff/LAeCgeQXajodpbT+b4zvNQ3/kkZ0TwspKIgDOnXMf0/MGDfkexU0Czobl5R3hbjIHuQQb1w8ic93c9FiFmDs4U0GlYtZCFxaahUq1Cjz+ePIZt1ltbTX3JW9wme8mqou/pzc3oubasFlBW63KlRBfmE7I6UJAQHgK7NOnk5rPaQwO7kSVu4onSaAVRp92qkPVYmKBOG9uN7xiJ0/j2DYheTCMidc+PMymSJYwMny5x1KuuYkaODDgZpGEJA4zsWOk9LqQZFi+HO8YeYayKYBDWR1S5kq7mTqhTLDYNFhXf0zthDxf229seW5srzRLZ/HZRa92Bn9zsEyPHAM2jr4v9ZKlW91Xa5mQJu66afH68o59kkSZCpubhFf2O652Of55ls+dl48e0g/T92IU7c7eS2i+d2l/uI1ZknY4RHiHbDCue8lDJ5bMD/3cY8eo+NIqOblhUypjx/mUAl8A30Fnd+yTZJxwriCkojR8pdyal2kziI08vHQJp1qCGDEMseMgJJsVN26ScQkR3jFjFHRRlTynFFcmS91OEdlOXSnU8QC2ZYFNbiw+u6gGHhxo+t3AgwOFxPOUAj8Qtk2A27Gr/2vNGq5frbZq1elNQZK2Nv0b5z0UHJkbihibm+/x3XaCyttGkVWubP0uwjwTMmdcY+vzXLOQzg/fZyNOZRKh0Mnis4tq8DcHo5puOJQCPwCxyiKaJtXgIB+GbsrRLQlz5+A6VXRK8PtsbqZFG9M2XbQAzYvQCGSlwjd8yWaa/o5EC/cda+n9uypucePTzmdpMw1l7f15UQr8AEgcuRIzUCcERBouv0Antf20wJBocGnE0lzbJUDzwNVv1+nCV3hLk8L5bNimMZaOGeeYz7bj+2zaWv9Z6LSNkbalFPgB4B6Qb9nDWJMqlF0i0WrbtfnYYOunb+UlpdwOZdt4xBCgPt+TYHGR16LzPr+iTkuh48v1MfaYc/eQtwavSRH0cf7umduT6/qlwA9AnpJlTe3U8i/S2EmwOrX5hPTTt7ZqpeJOnOYakxgbYBEngVhtZp8f5xQOYfLoPuVVMLJ+L1smzlBwLLVQ1pVSvKl3+jPTLU5b2ys0NbJSpcAPQghV01go3bFIJcIz76YhYQiJx6VAk4bPRjI9bd/AbPfpe5oIQQzGiMlvk3ez9THFhDJ5bKcRaRZLn0yctjF0XStks7Mhb3bMtF0/FKXAdyBGDn1rkRWLeUEiPGOahfIK61g+ibyCK8SkoMcrZgF1Di5mSgsTRCCIY2xGPuOWx5fhuv8QBSf90nRPnz6ZENuOHxqoZXqFohT4FsTKdx9iApIKz5iO37yCNsYCibHxhBTHcGn4efjaWUiZJdLv53nmafiMW9qcZrp/25hw9yMpgiLtY/akHOLjiGJyTSmGrmInndbwd1UunRD45Lu3QZpGuekzYWKymLVWQ9PlasQo9B2j8Ljrer6Jv2zjKc12moYruZg0X3wWIbV60+DGTefCSUMnaeNyHAH8XDLdP1Hy+zR0RllJH7PQY6ifD5e4zzZmeddWNrmazoOfF1NvnYrSThY9KfDTSc+41KY+hQfqS3U29en+of3s76TCM29GwDzIJt5605vswlSCGIXHXcVSbInTfMczZIPK5vY3QZIvPou8yfBsOeaz9QkAPuW1z/3rMc4Ke42VlebN07VZpnHlirsYi23M8q4tLrlaheRZBPcM7Nn+foUqmD40jfm758W/9wEp7il0AQ4dOqQuXboUtU29I7sy19X21XD5vsu52xusDOKxex4zZr7Tmkl6sg4Pt0+Yu2DqXxZEwLveBTz//E6FJp3HncOBA+bCKLVaoin69G92NmmrUkk0vFrNfX1f9PWZhRVRot26wN1vpZL8fnQ0SUe8sGAf61hzQ49b9nlx98lBev8a3DgArc8+3cf9+/mMs9VqUu2M67drzExjgZ+oY/bpWVy5fgWj+0Yxd3iOzVzZd6oPCuaLV4eqWFmTVVKp7atZr+MDIvqKUuqQ8bNeE/gHPnbAWbBgeGBYnIta0p5t8+AWX6eQ7k9fnyy/ffaorv+u1RJBdvFiZkGhuze6LPJuUJKNc3g4ST+cHivT2BU5PjaBbELIBj0xYf7MtXmMjJiFfrWanEy4DXVhwS7ss89l4K110HunsK523rTJg5GHR4xCnUAtG0Ef9WFL8TfpI3dssAn8njPp2Ew12Xz3eduTfCevTT0msrZqaTGTrM6g/87m8dd5zCcm+CIueesCF4EYZQbTZgNTzYAbNxLhnp4L8/PtnRs+ppTs/dueW7pWch8jcVymqnTN4jRWVvjnYxP2gNkUtPHfzjYJe8BSw3qpjpdeecnYtknrf+2rXttSTElynZjoOYHPFUKp7ath6+QWLt932WuHldScjF2Xsii4bKExoDcDUxGXEOdoOxDDh5Le2DlNNq9DNi8kfgegtRiK7bllPzPdu2TztBW6AcKej3G898nJF7NPz2Jjy1EyLoVra9e2iymxfYpctDyLnhP40pKFGlxxc1t70ra7De0WOFnHXwz2jg98ThMxT2IuZ30nTzn6Pm1CX6nkNKJhe262Mo4+wpmrJ6xUco2Q52N8DtdlNawBf+E8um8U4wfHcfm+y6zQL1o5zCXwiWg/Ef0JET3X+Pe1zPc2ieivGq8n81wzL2wlC7OwFTfXn2svvfayV4eqqA5Vg8xDnYatbKFenNPTzWXtuCO6FOlNRsreiSEQQ08TMa5tMxF1yynHZd6RPjdbGUcf4WyrJxyqqJjuceA/z2GQ7AqhVgJtzlpOqdS/Xb6+DAIZv1MoOIK+5AXgYQAfavz/QwAeYr63GtJ+p/Phu6rUxAjYkqKI/DWma/gEROVJoGUKcJFUdoqV2iFWts3Q6FfuecbKvRRjrkiDmWx9jhk0WETmWWNAmSXC3lUQZTu63pRmxfBbSfEUX6CoSFsAXwfw+sb/Xw/g68z3bkmBb8uYGSu5mgSLi/HK6kmuJRUWkoIiNmGfFZYSgR9r0berEpQvQiOZ0/mSJNGspt+G1qK1fSf2JpkniWCMTdCWK8eVlqVdMqNIgf+d1P8p/XfmezcBXALwZQDvc7Q51fjupdHR0agD4YvqQ1XjA6o+VHWmT45Zp9K3pmnRpwGfZGy2rJYhOYNi5T5xJf8yjV878qfHOnlIfy8V6K755No0Ys3HkLZibjqStOnc6d+WSiFWPVul7ALfycMnos8B+H7DR7MAFpRSr0l99x+UUi12fCL6QaXUt4johwF8HsBhpdQ3rBdGMTx8H3Ac2+pQFXsH9xr59zqAIhuMlYdja3JWaSwuNvP4TQE8MTnuMzPAmTPJsjEhey0j13kAuP32hGqX5pdL+O4xg7ZMsQCTk/z46SCvvNcO6Zft+Un48xzPPdZ4djOM93iwjsrPzWJrrzu4qqktJu4mHWvDfadCFWfqhRhc/Fw8fKXUzyqlftzw+hSAvyei1zcu8noA32ba+Fbj328C+AKAnwq8l+iwsXCurZnJv9fWrlnZPrHy80iQdfCdOVMc06Vetwt7E+MiS2nU3PuVlZ0+Hz2abCQSvnusvEIc1fLiRX78XNfWDl0ioL8/+dfXsTs+nmw6mqtfqTRTIE2QOC05h3yMNBcxUQRDqeVeDtaB90xhc6+ZjGGDhOXHBWJuqk0M9A1Y2y+ai5+XlvkkgMnG/ycBfCr7BSJ6LRHd1vj/CIB3APivOa8bBS4WDkeR0vQqju0TkkjNhmxSK42+vlbhxAnjGAt4dpZvn4hnXKQpc3v3AuvrzZ8rlWwkgJtPHSuvEBfhbBOAwZsy9wAAIABJREFUtmun2TXATtCaL8umXk9OGPr3m5vJ37bfu4KWbBtijGR4scAxlGZm8m0CLfdyeBYYDFPIXCy/+lK9hX2jUR2qgmzH9QaK5OLnSq1ARFUATwAYBbAM4H9USl0jokMAjiulPkhEPwPg4wC2kGwwH1NK/b6k/aJNOq7jmSlPjuTIJTn2+aBeB44dAzZSMR4DA81/uxDjiG7LteIKY9ewzfd2mRFsZpNQs43LrCK9Ny6FgO33pvtJp7ewpWTwMSEVkQZEksojm7rDx0RZryfZOJvG9GQfQK0TmUDYOilPDqRp2emcO7NPzxrXPoGwf2i/KLdOqJzYvlZRqRWUUitKqcNKqTsapp9rjfcvKaU+2Pj/f1FKHVRK3dn4VyTs2wGXJu7D2U/DN7jLhfFx4PHHmzVL/bcEoamUs7BpfZubMk3WlFZAI+QUEmICsAUKhZqMXH2X3Fu9zicJs/3edPI4dy4Rki6eu/TEVER8gDSVR1bJkJoodfvZMe17WR5cxbbNWAc4c46CYk3EaRTNxd/VkbauKFmbyUZDR8b5pF0I3SisbRoiCcfG+O9rTTpPKuWsMB0b88vxbhLGtvw8vmaEUBNAiNkGsLfp6rvk3mxCTCn7hpYnEljy2yKioPOk8pBsoFz7r/3L/AoZ56fj0iJXqMKmSq9QpX2Bmhx9pxteeXj4ksCoooKnYlIybXDx3NNUPBedzVRP1URl03VWbde18a85iqmk1qn0/l08dF/qYygX3ZcCKKn0FKvmru5z3piLPJTUkKplrmcl7XPeNWorZdj/YL/x/cHfHGwpZF5EsCZ6scShNMghtnBuZwSua8HoxSgppJ79PG8hcFvsACcYfTnaPgIju/nFiCjOCp10fIKOUPW5p3aVOdR9LWIMfBAaqS3d9IoMlHMVK+c2hOpD1cKVwZ4U+LYduMjB5iZC5VQl+nWlGr5r4vssPNsmIv19nqhQn/s39VujU9qt6SSl/65WlRoY8L+XEMQ85YQGVknnULUar/3tPudU9FwpFmyyJ9tO7A3AJvB3rQ3f5oBRntxbH3CO4E21Gf26rjJ/2tHo4lr7OEu1LVrbun2xf3/y27m5xIGrVPPnPnbhuTlgcFD2XVPZSKnNez9TpTKGzyFdL2BlJfEb7NljbycGZdKXf2/zbYQ6c00xGtnnqcsvSp5V1mcEMH3+CTsdWwLtp/PF6L7Rbd8inSIcPX80Vz98sWsFvittMVBMkIPE02+7rm/K3nT+cs2AyTpqXVxrV65xjSxbZXycZwrZsmhKi067xmJ8HHj1q/nraBAlgiiEw12vAy+/3Pr+wIA/80nipFxfB773Pf7zohlX6fdNAjQreDln7okTsnmc3nivXgUeeywsxoJz4Bv7HCkwcvzgOOukJVCL/BmsDOLqjauYOD+xzeZRmYybRQde7eoSh2mebHZgNXy5t5JrSmrmmq5bVI1brsRetZpoT1/8YqJpZnH4cHOtWp9yhZxgI0p+6+Ksz83JxsK3DqvveHL8+mo1EVA+8O2rCYuLcVJkuOaadC5K76noEpY+KSK4OrQhsmDmwgweudS6eKYPTeMdo+/Ylj/7h/bjpVdeEhVMySuTerbEYZpS2a6CA1lKJqcBmK5bVAEQfRLIRuyurCSL+oknzL97/vkd7WhuLgmsMmlQpmMzp/m7hL3WYLmxmJho1hilp5N0Gz7jyZk4rl3zjwGQmmK4WAU9pjFSD7j499K5KL2nogvZcHPK9PwkdGwp5u+ex/Sh6e11XqEKpg9NY/7u+Sb5s3dwr7g6VqFFUDjjfje8YqZHbnf+ep/rbqdOPUkK99UUDi6yjs48GQd9WRFp56DLySelddpyrAPuzJQmh6GUYWRzevrmpzexjVxOTImTUvfd5My2jWksLC7yDCvT2Pk472NmFZVe3+SE7oQssJFIYvcDvcjSMaFd/Hif6xq9/R8ebhL6NuHiA1/ec3qxWDnNDBsizUBJCz7bNTWkm9P0dKuA9bkvpcLyufvQTrMUWNuGZxL+6ZTNUqGWvT9J6uJqVan+fr+xM7XPjU3MugEaLnowt0baLQtsNM7YRVBKga9kD7gTGwI7Ee6rOYWL7wIK0VZdvw2paiT5vo/mOD0tax9IaI9Zgeeq6mQSmEVsnqG/s2nOvpuZ7SUtusO1W63GPY0olRqTg4vJmkmdkosoEBQKjsZZfahaBl7pVyyB38moWxfYo95JcgqX9EKXFqkI5VLbtHhOQHCCSBr04yq0ol+VSuvvuDFzlUuUCtIizGOhv7Nt/CEbNffKbpY2cKah2CaoWk0lwv7DmdKB/6r4teuLdimUPS/wJVG3scuPSR+uqG81+0L3iZrM4wuQ2OltgihrPqhWZf2QaKJSs46P4PWtMiU9jflo1nkihJWyKwshqQ18TpVFRrpqLC4qRf/CvIYqpypWoRoqgGMJ7qI2AJvA39W0TA0JDUvynTTNUydCurZ2raVijk9aZcl3XRS5TlUtsqUEJgKOHwfm55O/TZWyfNPcTkzw1xoaMqcHziI9JjZKoaRvpnTBgIzOWK8nhU1syeS43/mkKLbNDcBdKSsLrnKWCdz4+rQh6tOpPoChXedZdyaE/k7azuSdk7j43MWmlMu+ydRstMyeEPhcfnqNClUwNDCE1fXVls90buqZCzM4c+kMy+dPP3TffPimvNotE9Sy0Nu1sEzt26AFFpBUtXIJYBdmZszxAnv3Aqutj86ZR50ThtLc/lnoZ7S8nLSxuWnPR2/bcFx57H36xG1AgLn85G23mcdT9+vyZdnG0y5FxLW+0+tOrzXu+65c9LFqXXDtEKhJxoRsJj3Lw9dwRd1uqk2jsAeAsTvGUF+qW4U9kETInXjqhHXycWkXJCmYbakAQkP/85STq9fdAl9zr22VsnzSOszPA9PTzeX/pqeB737X/H2l7FGbXO77UGGfrXilYwq4trjnowWitA8zMzslFfv7k781bHx702ePP55EFi8u8nUBpLnxY5Wj5KBTFCxfX2arTAE76y6dw57D8vVlYyr1bFvS9119yiIrY2JH3vaEhg/Iom5N0AFbtkni01aohs/BVA0LSHKSPPaYWWgYqwDBz8TiqvCkoTcFmyabV9vLo0nGquIU0ocYkdXcqWd6esecFgpubHzutYgqWYA8oh3YWXeuk0AanGZdtIZvgm/kbc+bdLKgUw7VNP3dhubgs0mYENuWqOEb+s+lWdCQCmBpSL3NVkyUVGYq0mxRVCh/FqFmtaxA1OkrpGah/n6zH6BSAW7e9L8PiYDulAkxjRCTCOens6G2rxbsn7MhZMOSoudNOtnKV30kv+3RfaO5Q52rQ1VW2E9+YjJXIidb6L8JrgReUhOLJKReH99NR3vt1I0hkGMVNs8DbjyUkicPS6evAGSF0Dmnb/Z9iflOaqoJLXyex4SYhc0kwlWaC1nH2QyWsarZSVOwEChqycO8Rcx/AcADAH4UwNuUUkZ1nIjeDeA0gAqAR5VS/0bSfgwN37ST9qEPW3CrInrnBmDcjfcM7MGNjRsY3TeK1fVVtkCxaYd2OYGlxzhfU4JLM5dq+CatemAAuP32ZLPJaochR/uizAFFwHVykpw4XGYy07ORaPjSE5B0LoWcqGKcwtKmzz7qw6ZqvXG91kxmUqB1HQ8PDGOof8hZXDxvYXEXbKcPddJPRhep4X8VwBEAf2q5eAXA7wG4C8CbAfwSEb0553WtSGv0Jg16C1vYM7CH3VWBhLmjd27Trr54ZBGrH17ddrSevus021ZWG5E4gTltJHtaGfv1updjzKaF+TjUOIff1atmx7JvzdUiimZLEaKJZlNVZ6Ed2La2Qwqh6wR2tvelidCkOfJNeeyHhhImFjdeeRMDZouGm4S9rks7c2HGmGcegFE7P33XaWcqdV+nrC+49c4lfQxFFBs+EX0BwK+ZNHwiejuAB5RSP9f4+zcAQCn1r13thmj4vumJ86RKzWoRK2srVmqnhsth42vvn3ztWVx8aFykCbtSJdt+l0fbjsUfr1SSTaMojT+GJuri9/vGU2hwp6+ZmaSNzc1kfKammh22Upt7kY7nvHZ/bs1UqIIttdWkxR89f9S4pm1ael66Zl7E8g0Anbfh/yCAv039/ULjPSOIaIqILhHRpRdffNH7YqbiBiboHTU0VWpW41i+voxXbr6CwUpzyR6tdaQ1c1dMAPeQucINF1+ZFWvPJs18cTHRzF2bRKi2rZlE6d8fO2b/Padtbm7utDExAYyMxNX6Y6So5k5RlYq9bWkFsyzm5xPzjVLJv1l2js3mnj5xrK4mZjnpdYH8aZSl6ZU5DXtLbTXRmWefnmVPzjYtXVOjF48stmj7eg2HIHsi5+iesXwDLjgFPhF9joi+anjdE7UnDSilziqlDimlDr3uda/z/r3k6JV+gBxHf3V91VpqzCR8N7Y28OrBV7c8NABNm4MNU2+dYh9yLA6wr3klrxA8caKVNrqxkbzPQSoIdE7/WEKf22h0xSyJmYfjn7sqfEkrmPnC1J+BgWSTn5hoLbFYrcqd31IzUF5OvlQxs60FidM2puA1KYW2EoaSeJy8cAp8pdTPKqV+3PD6lPAa3wLwQ6m/39B4rxBwD7VCFeMD1A+4OtRcHWRlbcX6cLiJdW3tWstDk546AODicxfZz2IWbshCa3o6eIdoR7D51j/NIsv3d70P2LXdLG7cSBg/MWDbaKQnFI41ZCsKk/7t5cs72rpSfkFYkv5ogW4KWFtfTyKXTcqAyf8g1dx9mVQtvqo7xkSaN7cWfNgusQRvrFKKMdEOk85fALiDiN5IRIMAfhHAk0VdzKSxDw8MY+HeBfYBjh8cx97BvS1tmR6Onoicpm6acD4auO273L3lpW2ZokSBHdNNrCLePsgKCK4KlMbqanOEaSikG43rhGI6Rc3NtZpMQmrjhiDdn717E8HOwbSRc2a9sTG55i49WZo044VnFjB556RT8zatEQLh+KHjhWjMNnBrefn6stPEUxRyCXwiupeIXgDwdgAXiOizjfd/gIguAoBS6iaAXwXwWQB/DeAJpdTX8nWbR+iRTGIucYVmm4RvfanuzfvnUJSdz8bN1+/nOY5nSyu63gdanbxTU25BrPPD2CApjJ7VRDnYTigcsukoXOkpioDrZGbayDmz3hNPJAwdjWo1fwwE66t67qI7BYlhjZw7cg7zd+cMOw6AbS1LTDxFIJfAV0p9Qin1BqXUbUqp79NMHKXU/6uUGkt976JS6h8rpf6RUqpwfcbnSOajsdtMMybhqzcIE4WMgyuXRxF2PpcAWFlpXtR79rhpeGmcPp2kekhjcDB53wSTNrmwkGSXtG0S+mTCCXWp8zmricbC7GyrZr2+3uoLiRWgxLVjO5kRJVp7FtwcWVlp3vjW1sL62nQtT19V1vwDwHuNSJ2rPnDl8ALab+LpydQKGi4KZ5YW5Uvh9MmX4bp2kXDRAblUwxqSoJt07h4XBdRFD+QofjrLJUcT1JksuXY5jIyYtXkufQUHCTUxVpoI3yyZaaS/p09ZfX32VM5phORH8gmqyv5OQme05auKSYm03VfeIEspOk3L7Fr4auy+TtM8wRrt3PltdmuXsAfsjB0teHy0QJeTmHPQTk3ZGUWhDBzfEwoHiYMzBi3U1Y4kSOzEiebTkFTYA34ZUAG/oKosJI5RF1umSOdq+kTOBVEpqLbZ83ta4HMCmUDGo6Cv0zQve6bI6L70EXb2xQOY/O26kQ4oPQByizxEgLkEI5cmeX7evllw7RLZzTzj40nm0bRdn8tEqpE2p4yMJK/l5VabfdYXImVEucw+rna02YrzIaysmE8AlcrOGHDmNV9nPqd4ccw6SUxLeu24BHosurMLNhNPu+z5PS3wfTV2X6epxIZnS+8Qg25pgpEF8Q9TmPt0vYUOaHNaNvWV6WoIpVPC2eaCjWybBZfELbupmTYkn9iFrK8gbedWakfImqiJklOAxBchpUv6CuetrZ0xOH3az5nP2cmlQVW6DUlMS3rtuAR6kXTnNNLyw4R0TY2iWDw9LfBDaI4+TtPsA84WaRgeGLY6dGNmyUvD5wgroSmG5O6xCZo82S9tm4Wp3RiFWbJwZSTVhVlMG4dks5OcmqSBTtz3JNq7z3OymVV8BK4kpmWwMojV9dVtoanLkXLtF0V3NkHLD65gy8raijhQKwQ9LfDbEc6sH7A6qXDuyLmWa3G7fXWoWpjD1ucIa1rU09NyYRwaYekbDWzrb7p/2XYlwVC+rBnJZsF9RyJEJacmqTDmvifV3qXPyaZk+Ahcm5mFQKgOVaGUwsrayrbQfHn9ZQz0NQdApNtvV1qDNKSnh7LiVSTkqTIVux9FMQQ4xKraI0U3pzl2sWJCWDMu1hOQr9JXSJKzENiem+/6cTHcpO1xc7c6VMXV+69aP987uLfj613DpwBKzIpXPaXh15fqGHl4BHSKMHF+QnR0kvBz83B4O6FdtPMIC4Rr6+2ASxMOcTq7zGB567rGqBUrObVwz41LP2yb9y6zjdRUOnd4riVBIZCYQmYuzIhTngAo1FbugmndZ9O7aMT0JfSMhl9fquPYJ49hY2uD/U5Ww5Vo3+3U0GOeSrrlhBOCdp4YpNz5bH+Anfd0agpTYZhQ5BkD06lFVyBz1cGtL9WD0w/HWicjD48YC5YQCPuH9hs/S/eNTTN+5yQuPnexY2si1hj1XE1bkzCz5brWyB6dJKaPdplHOmH6sfXFtVkUtaH4VtrKC5f5pBvq6fqCuydTjeHsc7RVdnOZHrg5Ecs8BCSmm7Wba9Z1ElIPt12IsW56SuBzgjGkWLAksjZPARUftNvuzqHTpx6JfVwqcCVaskugt8ueHhO2Ai3pfvvYmYGwuRgyV2wR7ATCuSPnrELTp5h5u9dXDPSUDZ9jA9j47oBfqtX0++3i8LYrOMQFCaWzyMhFl7AHZJGpPnl1bDb+vKmj8yIk746NEpvut09a79Bi2yFzZe7wHEtrHN036vQHcDRNE4pYX0Xk7ZFi1wl87gFtqs0WapZGdajKplrN/magb6BpYheWsjgzKVxc4rztSycdp1ml3y9qc6rX5dklXQLXxxnLOS/r9UTQmpCtJpUnCRoHn0pk6ee9+s8PAAfNnUlvBtLnlSf9sC2FMIfxg+M4fui4Ma4lNvEgtuLmWxQlNnadwLcVA378fY83ecKrQ1UsHlnE1fuvspOVMhIm+3cRLBvTpHjplZfY8okx2pdOOu6klH5feurx3XRmZ+WpHjgtVgth7qQg1cy1sDXlmBkeTjJOFl2EfXvTOlgH7jsAnOzDjakDOPFo80Wyz3vl5jL6753aEfqp36/+853nwD3H6lBVlH5Y8nxtBUts82H+7vmWuJbJOycx+/Sscz5dW7vGtptGERtIp4ui9IwNP0QId8pu7sMlBuDt5MlzX3SKV7HVyWQuFWXnt9me0+Bs+FwB9zSktndu09AZO0Mzc/qgrw9QP14H3jMFDKZuan0Yi7/odlJW+2vA5+aw8jPNv9fPAUDwWvLJYhnC+gm9HsCPx97BvVjbWMOm2kSFKph661T0PPrt8Pn1lNMWCPd0Z39nO1Zq4VYEpJMidHPLM+mkm4XrGfhsOrqt5e9cAa6PAk/PAUvm+9MC1+SwdTl8fdg1LrqmhM6ZFwcOAMv3HgBeY1cObGl5XTTG0LXk83w5JcJHCPrOJ4kzugiWTjuUyJ4T+CHwYSRUqIKbH7lZWF+kkyJ08uSZdLFOUHk2NawPA58+2yL0XQLbxU7xoXO62DntYO/U68DEc30Aha3hvYN7sbq+yn6eR6nxUSpiCEFfJWZbibi+3ELHtPUhL22yHfTqnmLpSJG1L5546oSYkeBTwSoEUkdwaM1Mrv2xO8acNtdYPgupnd/IFBm8ARyeRbW6U5BbkmCNs+tzicxscEW7xoiGdWF8HKgOhDsVbcLeZUN3QfJ89RrUQjcNX/t5SObby/ddRm1fzUrR5Eqcat/XxPkJjDw8Ih6rTkTWp9GTAt/04LhgEhNq+2qFUqukkyK0Zqap/ck7J7HwzILIkRujzOLYHYY6egDetP9NTePK8q1fcwVXryYVp6QpGzghPPbr/s9SkqQtNOOnD06/152COwQKKpcj0aW0ZOtDK6htoR8iBEPZci4mkqTE6craCo6ePwo6RS3zxyQniihTKkUukw4R/QKABwD8KIC3KaWM9hciugzgZQCbAG5yx40sijLp+JQeNEXfaeHY6ahXqRlKcjRut4NaGu3IHbdD+5UNthr79ToW/qE7IphD4RMN6wPfxGaufqV/V8R8C+mnTRZIS5xyvwPCnd55UJgNn4h+FMAWgI8D+DWHwD+klPKoAFqcwPd5cKb8GlyaBl97X4ww6lg1M9sVMey6nglFhry3c6NrV/4ikyLAbZwu+/Xc4blChFa75xsHTmmqDlVx+q6khqWr1q4J2mneCZZfYTZ8pdRfK6W+nqeNTkDKLz77nrOYv3u+5fglCSxycd3zBmDoo+LR80cBYJuT7HO/6Xa4RV9U1S2fdhVUYTbPdkUwtzPgxmSyO37ouNHkcfzQcWOWRm0OKYo33q4IdRdMY6VjcwA4a+1y0IVMTGh3dHwaUVg6RPQF2DX8/wfAPwBQAD6ulDorabcoDT+vpzxGUrUimDK+piaXSajI46ePFlqkRtQuDb8bciHZThjcZ0Vp4t2UDJCDj+nXhApVjJtEV2v4RPQ5Ivqq4XWPRx/+qVLqLQDuAvA/E9E/s1xvioguEdGlF1980eMSckidopxjVuIgcmmOeTRLTuu6+NxFLwaALVdKkewBLVzSOY5sWmjsNBVprbpdtQFinSRM9xKDQMA5En01cWlfOs1WkSCvJr6pNttad0KCtmj4me8+AGBVKfXbru+2k4efhUsDyRtYlEfji6V1dcKOmndcY19Pf6do23oMDd90LwN9AyAirG+ub7/HRbSGaNQ+v7sVtHYf5NXwtQ+k3XUnCg+8sgl8ItoDoE8p9XLj/38C4EGl1H90tdtJgZ93gUoEW+jiiGUeKMrMEMLOqFAFC/cudKWgjYEYwtBHAEmD9HRpQFff089z7I4xY6GQbmHexIJveug0OrnRFea0JaJ7iegFAG8HcIGIPtt4/weI6GLja98H4P8komcA/DmACxJh32nkPYK7jqx5jrSxzBB52+HMCzbnpC2baRFOTFtwWjvT0sYwYfiYGLLf5X67srbiHIe0uWfu8BwbryHJpOoD21xqR4rh9DNzoTpURXWo2rXmKY0ytQKDbtEMTagv1XHiqRPbXGtNIQuZYHk41iaNdah/yJqbxaWltstZCtx65oYiNHzTd0P6UB2qstx/SSoS0zzk6M+SqlZFwJY4cPHIIkuM6CaTTk9G2kpQhDMvhlaiBW16ca3dXAvuU2jUH+c45ha91jBN42r6XizYrtfOtLQxYLqXgb4BUdps27z1GXNu07AFernojJwmb7uWae5NnJ8otKCITdM3nU47nfvehJ4X+JwQjs0ikPDyJZtBp/Npa/gKZs3q0OPK5dV38bB9N019PQ6d5ET7wjQnH3/f43jsnsdyzVMp972+VG/JeSNFyJx2VakzoUih6qs8dMtaTaOnTTrtZBXYnJVTb50S8+cl+ejTyHOkDHG+mjBYGcRj9zzmZI0M9A3g9ttux7W1a9sVvq6tXWvK/d9pR/etClsqi3NHzuWKP7GlWU6De1a2qOtsPWqb2TCNop5rfamOifMTxs+y7LZORROXJh0GeXZgX03T5qx85NIj4n5IKk7pvtEpwtHzR61HSu4+XCcSk7bDaX+vHnx1yyLPOsQIhI2tDaysrSRVmdZWtv+vr23KaHpj4wYmPzHpHH9fE107nILtBDf/FJRYAbC1cfqu084kbtyctlWpM52yJdeKeXKbuTCD/gf7QacIk5+YxJ6BPcbvZe+jW6KJ0+hpgR/KxAmxzYU8ZFM/OHuoft+UhTCN9KKz3YdkMxzqH9r+f3WoymppXEm58YPj24LYlVfH5h+QMHx8THTdaHtNI2QzsglVKWxtZMeXg2lO2zZjk49Jwp6JJVRnLszgkUuPbK+vTbWJ7258F30Z0cn5Tbot8GpXCnzpguAmhYKKbkd3OSul/eMmuX7fFj2roRed7T5smyHnODblZOHuQ0PSXwkkJzOpg7obba8aoZtRDOEzd3iuxUE8WBncbiM9vj55nUL8Zfpai0cWCxWqZ7/C+H8Izv5yacgldXeLwq4T+KYFwRUpsAlh20IKORm4nJVZcJPWtXAldnW96Gz3YTuOcgJR90VyH+lrSdFH9uka6xjfroRqIeDG/sRTJ6xKTiwSQtbnx/kAfTeY7GYMoCtSNHAn6i21JVIepDEM7cKuE/i2IgXZwXUdDX1tjtrRyGH84DgW7l1gj7wVqjgnrW2CS1gU6UXH9Xf/0H7rgrUF8fguPp+j95ayO7piHeO70faqYRt7lyDJW3hj9ulZbGxtNL23sbVhXCOueZoW5jMXZlr+dp1i0m3MPj2LucNzhRQUkfjMpOiGk+OuY+m48qxz3nsfj3p9qY5jnzzWMvlNbBQTZi7M4MylM9FzvEuCmtJMm5GHR4x2cR1uz7F0YjA+NLjw9b2De3Fb5TZcW7smykMek13VzTlhfIOvYuZyicE6kaQrcGVNbefz0Tb8LKYPTWP+7nmvtmyyiQvcCkFPsXRcWhinIflodeMHx3H7bbe3vL++uS7arefvnt/OXx/zGGozOaiTqkX74Zyp+n1OI5w7PGc8SYSUxePykb/8Gy/j6v1XsXVyy6rZF3GM7+ZMjj6+oHQAUwwTQoyTj8RnwwlFie8pNubvnsf0oeltjb5ClSBhD9jHqV2mnV2n4bs0CE7D99UauqViTxq+XPM83HQuHqCI++91Dn0W2ZMXV9Iwdj72GJq1T6WzLHS/u3HtSRAqm3zRUxq+1s5sVXxsv5Nqdd1o5/V1lOVhboRU1wpFN9LdmsIsAAAFdUlEQVTbOonsycvESx8eGGbNYKHO5xgnH+n8yJ4g08+7G9eeBN0Q9b3rBD6QDOzV+69i8chiENVL4vzpRiHkuyC7IWOnBN1sYukGcONTxKac1/ErMUnp0ovc8+7GtSfF+MHxtipLWew6k0470clc3d2AXr//bke3Op+l+fV92riV5l7Rz6XwAihFodsFfokS3Y5bWTDuZhT5XEqBX6JEiRI9gp5y2pYoUaJECTNKgV+iRIkSPYJS4JcoUaJEj6AU+CVKlCjRIygFfokSJUr0CLqapUNELwKQZYoqFiMArna6E564FfsM3Jr9vhX7DNya/b4V+wy0t981pdTrTB90tcDvFhDRJY7m1K24FfsM3Jr9vhX7DNya/b4V+wx0T79Lk06JEiVK9AhKgV+iRIkSPYJS4MvAp7jrXtyKfQZuzX7fin0Gbs1+34p9Brqk36UNv0SJEiV6BKWGX6JEiRI9glLglyhRokSPoBT4BhDRLxDR14hoi4hYKhURXSaiJSL6KyLqaFpPjz6/m4i+TkTPE9GH2tlHpj/7iehPiOi5xr+vZb632RjnvyKiJ9vdz0YfrGNHRLcR0R81Pv8zIjrQ/l629MnV518hohdTY/vBTvQzCyJ6jIi+TURfZT4nIvqdxn09S0RvaXcfDX1y9fmdRHQ9NdYfaXcfoZQqX5kXgB8F8CMAvgDgkOV7lwGMdLq/0j4DqAD4BoAfBjAI4BkAb+5wvx8G8KHG/z8E4CHme6sd7qdz7ADMADjT+P8vAvijW6DPvwLgdzvZT6bv/wzAWwB8lfl8DMBTAAjATwP4s1ugz+8E8JlO9rHU8A1QSv21Uurrne6HD4R9fhuA55VS31RKrQP4DwDuKb53VtwDYKHx/wUA7+tgX2yQjF36Xv4YwGEiMld7bw+68XmLoJT6UwDXLF+5B8AfqARfBvAaInp9e3pnhqDPHUcp8PNBAfg/iOgrRDTV6c4I8IMA/jb19wuN9zqJ71NK/V3j//8fgO9jvvcqIrpERF8mok5sCpKx2/6OUuomgOsAqm3pnRnS5/0/NMwif0xEP9SeruVGN85lCd5ORM8Q0VNE9GPtvnh/uy/YLSCizwH4fsNHs0qpTwmb+adKqW8R0X8D4E+I6P9u7PKFIFKf2w5bv9N/KKUUEXE84VpjrH8YwOeJaEkp9Y3Yfe1BfBrAHyqlXiGi/wnJCeVdHe7TbsVfIpnHq0Q0BuCTAO5oZwd6VuArpX42Qhvfavz7bSL6BJIjdGECP0KfvwUgrcG9ofFeobD1m4j+noher5T6u8aR/NtMG3qsv0lEXwDwU0js0+2CZOz0d14gon4A+wCstKd7Rjj7rJRK9+9RJD6VWwEdmct5oJR6KfX/i0Q0T0QjSqm2JYMrTTqBIKI9RPRq/X8A/x0Ao3e+i/AXAO4gojcS0SASx2JHGC8pPAlgsvH/SQAtJxUiei0R3db4/wiAdwD4r23rYQLJ2KXv5ecBfF41vHUdgrPPGbv3ewH8dRv7lwdPAvjlBlvnpwFcT5kGuxJE9P3ap0NEb0Mif9urEHTas92NLwD3IrEJvgLg7wF8tvH+DwC42Pj/DyNhPTwD4GtIzCpd3efG32MA/gaJdtzRPjf6UwXwNIDnAHwOwP7G+4cAPNr4/88AWGqM9RKAD3Sory1jB+BBAO9t/P9VAP53AM8D+HMAP9wF4+vq879uzN9nAPwnAP+k031u9OsPAfwdgI3GvP4AgOMAjjc+JwC/17ivJVjYdF3U519NjfWXAfxMu/tYplYoUaJEiR5BadIpUaJEiR5BKfBLlChRokdQCvwSJUqU6BGUAr9EiRIlegSlwC9RokSJHkEp8EuUKFGiR1AK/BIlSpToEfz/0Zo7/VigsXsAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c7X4J_pJkCJJ",
        "outputId": "b87240f3-680e-4413-c392-48e39198c48e"
      },
      "source": [
        "#output_data = 2*output_data -1\n",
        "print(input_data, input_data.shape, output_data.shape)"
      ],
      "id": "c7X4J_pJkCJJ",
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tensor([[ 0.7582, -1.5591],\n",
            "        [-0.5533, -0.5972],\n",
            "        [ 1.5560,  0.2821],\n",
            "        ...,\n",
            "        [ 1.4986, -1.4743],\n",
            "        [-1.5421, -0.9109],\n",
            "        [-0.8700, -1.6964]]) torch.Size([1000, 2]) torch.Size([1000])\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "id": "qBGOEnxlkCJK"
      },
      "source": [
        "#we should use one hot label embedding with MSELoss. Hence it is necessary to implement a Softmax\n",
        "#Maybe we should also implement CrossEntropyLoss\n",
        "#print(input_data.shape, output_data.shape)\n",
        "#output_data = convert_to_one_hot_labels(input_data, output_data)\n",
        "#print(input_data.shape, output_data.shape)"
      ],
      "id": "qBGOEnxlkCJK",
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rp3d4oV9kCJK"
      },
      "source": [
        "#handmade sequential linear + relu \n",
        "linear1 = Linear(2, 25, True)\n",
        "linear2 = Linear(25,25,True)\n",
        "linear3 = Linear(25,1,True)\n",
        "sigma1 = Tanh()\n",
        "sigma2 = Tanh()\n",
        "sigma3 = Tanh()\n",
        "loss = MSE()\n",
        "\n",
        "net = Sequential([\n",
        "    linear1, \n",
        "    sigma1 ,\n",
        "    linear2,\n",
        "    sigma2 ,\n",
        "    linear3,\n",
        "    sigma3 ,\n",
        "])"
      ],
      "id": "rp3d4oV9kCJK",
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wbfKVjy-kCJK",
        "outputId": "5b7600a9-3fe9-49df-e33f-a1d74112cd5c"
      },
      "source": [
        "optimizer = SGD(lr = 1e-4,max_iter = 100, parameters = net.get_parameters())\n",
        "n=10**2\n",
        "N=output_data.shape[0]\n",
        "for t in range(n):\n",
        "    optimizer.zero_grad()\n",
        "    acc_loss=0\n",
        "    permuted_index = torch.randperm(input_data.size()[0])\n",
        "    input_data_shuffled = input_data[permuted_index]\n",
        "    output_data_shuffled = output_data[permuted_index]\n",
        "    for i in range(N):\n",
        "        x=input_data_shuffled[i]\n",
        "        y=2*output_data_shuffled[i]-1\n",
        "        # Forward pass: compute predicted y by passing x to the model.\n",
        "        y_pred = net.forward(x.unsqueeze(0))\n",
        "        # Compute and print loss.\n",
        "        acc_loss += loss.forward(y_pred,y.unsqueeze(0))\n",
        "        \n",
        "        # Backward pass: compute gradient of the loss with respect to model parameters\n",
        "        net.backward(loss.backward())\n",
        "\n",
        "    # Calling the step function on an Optimizer makes an update to its parameters\n",
        "    new_par = optimizer.step()\n",
        "    #print(len(new_par))\n",
        "    net.set_parameters(new_par)\n",
        "    \n",
        "    \n",
        "    if t%10==0:\n",
        "        print(t, '   MSE loss = ' , acc_loss.item())"
      ],
      "id": "wbfKVjy-kCJK",
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0    MSE loss =  1026.8297119140625\n",
            "10    MSE loss =  972.5297241210938\n",
            "20    MSE loss =  941.5697631835938\n",
            "30    MSE loss =  902.7471313476562\n",
            "40    MSE loss =  850.5894165039062\n",
            "50    MSE loss =  782.9170532226562\n",
            "60    MSE loss =  703.6536865234375\n",
            "70    MSE loss =  621.3328857421875\n",
            "80    MSE loss =  542.839111328125\n",
            "90    MSE loss =  472.0589294433594\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EveAJpA8kCJL",
        "outputId": "5b2c0cda-6f80-4fa5-9aea-7f607f59cc24"
      },
      "source": [
        "correct=0\n",
        "for i in range(output_data.shape[0]):\n",
        "        x=input_data[i]\n",
        "        y=2*output_data[i]-1\n",
        "        # Forward pass: compute predicted y by passing x to the model.\n",
        "        y_pred = net.forward(x.unsqueeze(0))\n",
        "        if abs(y_pred-output_data[i])<1:\n",
        "            correct+=1\n",
        "print('Correct predictions after '+str(n)+' training steps: '+str(correct/N*100)+' %')"
      ],
      "id": "EveAJpA8kCJL",
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Correct predictions after 100 training steps: 96.5 %\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zSqJHio4kCJL"
      },
      "source": [
        "#Reinitialize net\n",
        "#handmade sequential linear + relu \n",
        "linear1 = Linear(2, 25, True)\n",
        "linear2 = Linear(25,25,True)\n",
        "linear3 = Linear(25,1,True)\n",
        "sigma1 = Tanh()\n",
        "sigma2 = Tanh()\n",
        "sigma3 = Tanh()\n",
        "loss = MSE()\n",
        "\n",
        "net = Sequential([\n",
        "    linear1, \n",
        "    sigma1 ,\n",
        "    linear2,\n",
        "    sigma2 ,\n",
        "    linear3,\n",
        "    sigma3 ,\n",
        "])"
      ],
      "id": "zSqJHio4kCJL",
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "scrolled": true,
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ft_RT0hWkCJL",
        "outputId": "41c09430-8f4b-401a-feac-8663c1e301e0"
      },
      "source": [
        "optimizer = SGD(lr = 1e-1,max_iter = 100, parameters = net.get_parameters())\n",
        "n=10**2\n",
        "N=output_data.shape[0]\n",
        "batch_size = 100\n",
        "for t in range(n):\n",
        "    acc_loss=0\n",
        "    permuted_index = torch.randperm(input_data.size()[0])\n",
        "    input_data_shuffled = input_data[permuted_index]\n",
        "    output_data_shuffled = output_data[permuted_index]\n",
        "    for b in range(0, N, batch_size):\n",
        "        predictions = net.forward(input_data_shuffled[b:b+batch_size])\n",
        "        l= loss.forward(predictions, output_data_shuffled[b:b+batch_size].unsqueeze(-1))\n",
        "        acc_loss += l\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        net.backward(loss.backward())\n",
        "        \n",
        "        new_par = optimizer.step()\n",
        "        net.set_parameters(new_par)\n",
        "    \n",
        "    print(t, '   MSE loss = ' , acc_loss.item())"
      ],
      "id": "Ft_RT0hWkCJL",
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0    MSE loss =  2.797254800796509\n",
            "1    MSE loss =  2.191330909729004\n",
            "2    MSE loss =  2.127686023712158\n",
            "3    MSE loss =  2.063326835632324\n",
            "4    MSE loss =  2.0039925575256348\n",
            "5    MSE loss =  1.914603590965271\n",
            "6    MSE loss =  1.8399988412857056\n",
            "7    MSE loss =  1.7531986236572266\n",
            "8    MSE loss =  1.661909580230713\n",
            "9    MSE loss =  1.5706528425216675\n",
            "10    MSE loss =  1.5355052947998047\n",
            "11    MSE loss =  1.529868721961975\n",
            "12    MSE loss =  1.529868721961975\n",
            "13    MSE loss =  1.529868721961975\n",
            "14    MSE loss =  1.5298686027526855\n",
            "15    MSE loss =  1.5298686027526855\n",
            "16    MSE loss =  1.529868721961975\n",
            "17    MSE loss =  1.5298688411712646\n",
            "18    MSE loss =  1.5298686027526855\n",
            "19    MSE loss =  1.5298688411712646\n",
            "20    MSE loss =  1.5298686027526855\n",
            "21    MSE loss =  1.529868721961975\n",
            "22    MSE loss =  1.5298686027526855\n",
            "23    MSE loss =  1.529868721961975\n",
            "24    MSE loss =  1.5298686027526855\n",
            "25    MSE loss =  1.529868721961975\n",
            "26    MSE loss =  1.5298686027526855\n",
            "27    MSE loss =  1.529868721961975\n",
            "28    MSE loss =  1.529868721961975\n",
            "29    MSE loss =  1.5298686027526855\n",
            "30    MSE loss =  1.529868721961975\n",
            "31    MSE loss =  1.529868721961975\n",
            "32    MSE loss =  1.529868721961975\n",
            "33    MSE loss =  1.529868721961975\n",
            "34    MSE loss =  1.5298686027526855\n",
            "35    MSE loss =  1.5298686027526855\n",
            "36    MSE loss =  1.5298686027526855\n",
            "37    MSE loss =  1.529868721961975\n",
            "38    MSE loss =  1.529868721961975\n",
            "39    MSE loss =  1.5298686027526855\n",
            "40    MSE loss =  1.529868721961975\n",
            "41    MSE loss =  1.529868721961975\n",
            "42    MSE loss =  1.529868721961975\n",
            "43    MSE loss =  1.5298686027526855\n",
            "44    MSE loss =  1.529868721961975\n",
            "45    MSE loss =  1.5298686027526855\n",
            "46    MSE loss =  1.529868721961975\n",
            "47    MSE loss =  1.529868721961975\n",
            "48    MSE loss =  1.529868721961975\n",
            "49    MSE loss =  1.5298686027526855\n",
            "50    MSE loss =  1.5298686027526855\n",
            "51    MSE loss =  1.5298686027526855\n",
            "52    MSE loss =  1.5298686027526855\n",
            "53    MSE loss =  1.529868721961975\n",
            "54    MSE loss =  1.529868483543396\n",
            "55    MSE loss =  1.5298686027526855\n",
            "56    MSE loss =  1.529868721961975\n",
            "57    MSE loss =  1.5298688411712646\n",
            "58    MSE loss =  1.529868721961975\n",
            "59    MSE loss =  1.5298688411712646\n",
            "60    MSE loss =  1.5298686027526855\n",
            "61    MSE loss =  1.5298686027526855\n",
            "62    MSE loss =  1.5298686027526855\n",
            "63    MSE loss =  1.529868721961975\n",
            "64    MSE loss =  1.5298688411712646\n",
            "65    MSE loss =  1.529868721961975\n",
            "66    MSE loss =  1.5298686027526855\n",
            "67    MSE loss =  1.5298686027526855\n",
            "68    MSE loss =  1.529868721961975\n",
            "69    MSE loss =  1.529868721961975\n",
            "70    MSE loss =  1.5298686027526855\n",
            "71    MSE loss =  1.5298686027526855\n",
            "72    MSE loss =  1.5298688411712646\n",
            "73    MSE loss =  1.529868721961975\n",
            "74    MSE loss =  1.5298686027526855\n",
            "75    MSE loss =  1.529868721961975\n",
            "76    MSE loss =  1.529868721961975\n",
            "77    MSE loss =  1.5298686027526855\n",
            "78    MSE loss =  1.529868721961975\n",
            "79    MSE loss =  1.529868721961975\n",
            "80    MSE loss =  1.529868721961975\n",
            "81    MSE loss =  1.5298686027526855\n",
            "82    MSE loss =  1.5298686027526855\n",
            "83    MSE loss =  1.5298688411712646\n",
            "84    MSE loss =  1.5298686027526855\n",
            "85    MSE loss =  1.5298686027526855\n",
            "86    MSE loss =  1.529868721961975\n",
            "87    MSE loss =  1.529868721961975\n",
            "88    MSE loss =  1.529868721961975\n",
            "89    MSE loss =  1.529868721961975\n",
            "90    MSE loss =  1.529868721961975\n",
            "91    MSE loss =  1.5298686027526855\n",
            "92    MSE loss =  1.5298686027526855\n",
            "93    MSE loss =  1.5298686027526855\n",
            "94    MSE loss =  1.5298686027526855\n",
            "95    MSE loss =  1.529868721961975\n",
            "96    MSE loss =  1.5298686027526855\n",
            "97    MSE loss =  1.529868721961975\n",
            "98    MSE loss =  1.529868721961975\n",
            "99    MSE loss =  1.5298688411712646\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "al67_83jkCJM",
        "outputId": "7c5f95ae-790f-4bfe-ac6a-016f60ae6616"
      },
      "source": [
        "correct=0\n",
        "for i in range(output_data.shape[0]):\n",
        "        x=input_data[i]\n",
        "        y=2*output_data[i]-1\n",
        "        # Forward pass: compute predicted y by passing x to the model.\n",
        "        y_pred = net.forward(x.unsqueeze(0))\n",
        "        if abs(y_pred-output_data[i])<1:\n",
        "            correct+=1\n",
        "print('Correct predictions after '+str(n)+' training steps: '+str(correct/N*100)+' %')"
      ],
      "id": "al67_83jkCJM",
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Correct predictions after 100 training steps: 100.0 %\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YyXLmaUhkCJM",
        "outputId": "e1081077-b14c-4f40-c43a-d62cdb0510cb"
      },
      "source": [
        "#handmade sequential linear + relu with sigmoid to end and no batches\n",
        "linear1 = Linear(2, 25, True)\n",
        "linear2 = Linear(25,25,True)\n",
        "linear3 = Linear(25,1,True)\n",
        "sigma1 = ReLu()\n",
        "sigma2 = ReLu()\n",
        "sigma3 = Sigmoid()\n",
        "loss = MSE()\n",
        "\n",
        "net = Sequential([\n",
        "    linear1, \n",
        "    sigma1 ,\n",
        "    linear2,\n",
        "    sigma2 ,\n",
        "    linear3,\n",
        "    sigma3 ,\n",
        "])\n",
        "optimizer = SGD(lr = 1e-4,max_iter = 100, parameters = net.get_parameters())\n",
        "n=10**2\n",
        "N=output_data.shape[0]\n",
        "for t in range(n):\n",
        "    optimizer.zero_grad()\n",
        "    acc_loss=0\n",
        "    permuted_index = torch.randperm(input_data.size()[0])\n",
        "    input_data_shuffled = input_data[permuted_index]\n",
        "    output_data_shuffled = output_data[permuted_index]\n",
        "    for i in range(N):\n",
        "        x=input_data_shuffled[i]\n",
        "        y=output_data_shuffled[i]\n",
        "        # Forward pass: compute predicted y by passing x to the model.\n",
        "        y_pred = net.forward(x.unsqueeze(0))\n",
        "        # Compute and print loss.\n",
        "        acc_loss += loss.forward(y_pred,y.unsqueeze(0))\n",
        "        \n",
        "        # Backward pass: compute gradient of the loss with respect to model parameters\n",
        "        net.backward(loss.backward())\n",
        "\n",
        "    # Calling the step function on an Optimizer makes an update to its parameters\n",
        "    new_par = optimizer.step()\n",
        "    #print(len(new_par))\n",
        "    net.set_parameters(new_par)\n",
        "    \n",
        "    \n",
        "    if t%10==0:\n",
        "        print(t, '   MSE loss = ' , acc_loss.item())\n",
        "    \n",
        "correct=0\n",
        "for i in range(output_data.shape[0]):\n",
        "        x=input_data[i]\n",
        "        y=2*output_data[i]-1\n",
        "        # Forward pass: compute predicted y by passing x to the model.\n",
        "        y_pred = net.forward(x.unsqueeze(0))\n",
        "        if abs(y_pred-output_data[i])<1:\n",
        "            correct+=1\n",
        "print('Correct predictions after '+str(n)+' training steps: '+str(correct/N*100)+' %')"
      ],
      "id": "YyXLmaUhkCJM",
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0    MSE loss =  250.3065643310547\n",
            "10    MSE loss =  247.48760986328125\n",
            "20    MSE loss =  245.02781677246094\n",
            "30    MSE loss =  242.7385711669922\n",
            "40    MSE loss =  240.51614379882812\n",
            "50    MSE loss =  238.26028442382812\n",
            "60    MSE loss =  235.92877197265625\n",
            "70    MSE loss =  233.47219848632812\n",
            "80    MSE loss =  230.87245178222656\n",
            "90    MSE loss =  228.10482788085938\n",
            "Correct predictions after 100 training steps: 100.0 %\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DLdc4ZlWkCJN",
        "outputId": "82868939-5cfe-43e4-e50e-0a8aa8826f27"
      },
      "source": [
        "#handmade sequential linear + relu with softmax with batches\n",
        "linear1 = Linear(2, 25, True)\n",
        "linear2 = Linear(25,25,True)\n",
        "linear3 = Linear(25,1,True)\n",
        "sigma1 = ReLu()\n",
        "sigma2 = ReLu()\n",
        "sigma3 = Sigmoid()\n",
        "loss = MSE()\n",
        "\n",
        "net = Sequential([\n",
        "    linear1, \n",
        "    sigma1 ,\n",
        "    linear2,\n",
        "    sigma2 ,\n",
        "    linear3,\n",
        "    sigma3 ,\n",
        "])\n",
        "optimizer = SGD(lr = 1e-1,max_iter = 100, parameters = net.get_parameters())\n",
        "n=10**2\n",
        "N=output_data.shape[0]\n",
        "batch_size = 100\n",
        "for t in range(n):\n",
        "    acc_loss=0\n",
        "    permuted_index = torch.randperm(input_data.size()[0])\n",
        "    input_data_shuffled = input_data[permuted_index]\n",
        "    output_data_shuffled = output_data[permuted_index]\n",
        "    for b in range(0, N, batch_size):\n",
        "        predictions = net.forward(input_data_shuffled[b:b+batch_size])\n",
        "        l= loss.forward(predictions, output_data_shuffled[b:b+batch_size].unsqueeze(-1))\n",
        "        acc_loss += l\n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        net.backward(loss.backward())\n",
        "        \n",
        "        new_par = optimizer.step()\n",
        "        net.set_parameters(new_par)\n",
        "    \n",
        "    print(t, '   MSE loss = ' , acc_loss.item())\n",
        "    \n",
        "correct=0\n",
        "for i in range(output_data.shape[0]):\n",
        "        x=input_data[i]\n",
        "        y=2*output_data[i]-1\n",
        "        # Forward pass: compute predicted y by passing x to the model.\n",
        "        y_pred = net.forward(x.unsqueeze(0))\n",
        "        if abs(y_pred-output_data[i])<1:\n",
        "            correct+=1\n",
        "print('Correct predictions after '+str(n)+' training steps: '+str(correct/N*100)+' %')"
      ],
      "id": "DLdc4ZlWkCJN",
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0    MSE loss =  2.4857826232910156\n",
            "1    MSE loss =  2.464448928833008\n",
            "2    MSE loss =  2.446307420730591\n",
            "3    MSE loss =  2.427915096282959\n",
            "4    MSE loss =  2.411731719970703\n",
            "5    MSE loss =  2.3950605392456055\n",
            "6    MSE loss =  2.378262758255005\n",
            "7    MSE loss =  2.3612921237945557\n",
            "8    MSE loss =  2.3435561656951904\n",
            "9    MSE loss =  2.325162172317505\n",
            "10    MSE loss =  2.31282114982605\n",
            "11    MSE loss =  2.3122923374176025\n",
            "12    MSE loss =  2.3122923374176025\n",
            "13    MSE loss =  2.3122920989990234\n",
            "14    MSE loss =  2.3122923374176025\n",
            "15    MSE loss =  2.3122920989990234\n",
            "16    MSE loss =  2.3122923374176025\n",
            "17    MSE loss =  2.3122923374176025\n",
            "18    MSE loss =  2.3122925758361816\n",
            "19    MSE loss =  2.3122923374176025\n",
            "20    MSE loss =  2.3122923374176025\n",
            "21    MSE loss =  2.3122923374176025\n",
            "22    MSE loss =  2.3122920989990234\n",
            "23    MSE loss =  2.3122923374176025\n",
            "24    MSE loss =  2.3122925758361816\n",
            "25    MSE loss =  2.3122923374176025\n",
            "26    MSE loss =  2.3122925758361816\n",
            "27    MSE loss =  2.3122923374176025\n",
            "28    MSE loss =  2.3122923374176025\n",
            "29    MSE loss =  2.3122923374176025\n",
            "30    MSE loss =  2.3122925758361816\n",
            "31    MSE loss =  2.3122923374176025\n",
            "32    MSE loss =  2.3122923374176025\n",
            "33    MSE loss =  2.3122923374176025\n",
            "34    MSE loss =  2.3122925758361816\n",
            "35    MSE loss =  2.3122923374176025\n",
            "36    MSE loss =  2.3122925758361816\n",
            "37    MSE loss =  2.3122920989990234\n",
            "38    MSE loss =  2.3122925758361816\n",
            "39    MSE loss =  2.3122923374176025\n",
            "40    MSE loss =  2.3122923374176025\n",
            "41    MSE loss =  2.3122920989990234\n",
            "42    MSE loss =  2.3122923374176025\n",
            "43    MSE loss =  2.3122923374176025\n",
            "44    MSE loss =  2.3122923374176025\n",
            "45    MSE loss =  2.3122923374176025\n",
            "46    MSE loss =  2.3122923374176025\n",
            "47    MSE loss =  2.3122923374176025\n",
            "48    MSE loss =  2.3122925758361816\n",
            "49    MSE loss =  2.3122923374176025\n",
            "50    MSE loss =  2.3122923374176025\n",
            "51    MSE loss =  2.3122923374176025\n",
            "52    MSE loss =  2.3122923374176025\n",
            "53    MSE loss =  2.3122925758361816\n",
            "54    MSE loss =  2.3122925758361816\n",
            "55    MSE loss =  2.3122923374176025\n",
            "56    MSE loss =  2.3122923374176025\n",
            "57    MSE loss =  2.3122923374176025\n",
            "58    MSE loss =  2.3122925758361816\n",
            "59    MSE loss =  2.3122920989990234\n",
            "60    MSE loss =  2.3122923374176025\n",
            "61    MSE loss =  2.3122923374176025\n",
            "62    MSE loss =  2.3122920989990234\n",
            "63    MSE loss =  2.3122923374176025\n",
            "64    MSE loss =  2.3122923374176025\n",
            "65    MSE loss =  2.3122925758361816\n",
            "66    MSE loss =  2.3122923374176025\n",
            "67    MSE loss =  2.3122923374176025\n",
            "68    MSE loss =  2.3122923374176025\n",
            "69    MSE loss =  2.3122923374176025\n",
            "70    MSE loss =  2.3122923374176025\n",
            "71    MSE loss =  2.3122920989990234\n",
            "72    MSE loss =  2.3122923374176025\n",
            "73    MSE loss =  2.3122925758361816\n",
            "74    MSE loss =  2.3122925758361816\n",
            "75    MSE loss =  2.3122923374176025\n",
            "76    MSE loss =  2.3122925758361816\n",
            "77    MSE loss =  2.3122923374176025\n",
            "78    MSE loss =  2.3122925758361816\n",
            "79    MSE loss =  2.3122923374176025\n",
            "80    MSE loss =  2.3122923374176025\n",
            "81    MSE loss =  2.3122925758361816\n",
            "82    MSE loss =  2.3122925758361816\n",
            "83    MSE loss =  2.3122923374176025\n",
            "84    MSE loss =  2.3122923374176025\n",
            "85    MSE loss =  2.3122923374176025\n",
            "86    MSE loss =  2.3122923374176025\n",
            "87    MSE loss =  2.3122923374176025\n",
            "88    MSE loss =  2.3122923374176025\n",
            "89    MSE loss =  2.3122923374176025\n",
            "90    MSE loss =  2.3122923374176025\n",
            "91    MSE loss =  2.3122923374176025\n",
            "92    MSE loss =  2.3122925758361816\n",
            "93    MSE loss =  2.3122920989990234\n",
            "94    MSE loss =  2.3122923374176025\n",
            "95    MSE loss =  2.3122923374176025\n",
            "96    MSE loss =  2.3122920989990234\n",
            "97    MSE loss =  2.3122923374176025\n",
            "98    MSE loss =  2.3122925758361816\n",
            "99    MSE loss =  2.3122923374176025\n",
            "Correct predictions after 100 training steps: 100.0 %\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XehGZ6FBkCJN"
      },
      "source": [
        ""
      ],
      "id": "XehGZ6FBkCJN",
      "execution_count": 25,
      "outputs": []
    }
  ]
}