introduction_experiment/expeSurprise.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment 1\n",
    "\n",
    "We want to show that the differences do not come from numerical precision but from incoming zeros during back propagation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Protocol:\n",
    "\n",
    "* We initialize three networks:\n",
    "    - One using the modified version of RelU as activation function\n",
    "    - Two others using the original pytorch RelU as activation function\n",
    "* We set the same initialization on all networks\n",
    "* We train the three networks with the same batches\n",
    "* We keep track of the $L^1$ norm of difference of weight matrices and bias vectors between the three networks \n",
    "* We keep track of the minimal absolute activation of the first hidden layer before application of ReLU."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import numpy.random as npr\n",
    "import argparse\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from relu import MyReLU\n",
    "import torch.optim as optim\n",
    "from torchvision import datasets, transforms\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "\n",
    "## Double precision or not?\n",
    "doublePrecision = False\n",
    "doublePrecision = True\n",
    "\n",
    "## To save results\n",
    "bitsString = \"32bits\"\n",
    "if doublePrecision:\n",
    "    bitsString = \"64bits\"\n",
    "\n",
    "\n",
    "if doublePrecision:\n",
    "    torch.set_default_tensor_type(torch.DoubleTensor)\n",
    "    torch.set_default_dtype(torch.float64)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Initialize random seed\n",
    "np.random.seed(12345)\n",
    "torch.manual_seed(123)\n",
    "\n",
    "## Load Training data\n",
    "\n",
    "batch_size = 128\n",
    "\n",
    "transform = transforms.ToTensor()\n",
    "\n",
    "train_data = datasets.MNIST(root='data', train=True,\n",
    "                                   download=True, transform=transform)\n",
    "test_data = datasets.MNIST(root='data', train=False,\n",
    "                                  download=True, transform=transform)\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size)\n",
    "test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build Models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==> Building models..\n",
      "==> Copying models initial parameters..\n",
      "Sequential(\n",
      "  (0): Linear(in_features=784, out_features=2000, bias=True)\n",
      "  (1): MyReLU()\n",
      "  (2): Linear(in_features=2000, out_features=128, bias=True)\n",
      "  (3): MyReLU()\n",
      "  (4): Linear(in_features=128, out_features=10, bias=True)\n",
      ")\n",
      "Sequential(\n",
      "  (0): Linear(in_features=784, out_features=2000, bias=True)\n",
      "  (1): ReLU()\n",
      "  (2): Linear(in_features=2000, out_features=128, bias=True)\n",
      "  (3): ReLU()\n",
      "  (4): Linear(in_features=128, out_features=10, bias=True)\n",
      ")\n",
      "Sequential(\n",
      "  (0): Linear(in_features=784, out_features=2000, bias=True)\n",
      "  (1): ReLU()\n",
      "  (2): Linear(in_features=2000, out_features=128, bias=True)\n",
      "  (3): ReLU()\n",
      "  (4): Linear(in_features=128, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print('==> Building models..')\n",
    "net1 = torch.nn.Sequential(\n",
    "    torch.nn.Linear(28*28, 2000),\n",
    "    MyReLU(),\n",
    "    torch.nn.Linear(2000, 128),\n",
    "    MyReLU(),\n",
    "    torch.nn.Linear(128, 10))\n",
    "\n",
    "net2 = torch.nn.Sequential(\n",
    "    torch.nn.Linear(28*28,2000),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(2000, 128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128, 10))\n",
    "\n",
    "net3 = torch.nn.Sequential(\n",
    "    torch.nn.Linear(28*28,2000),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(2000, 128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128, 10))\n",
    "\n",
    "if doublePrecision:\n",
    "    net1 = net1.to(torch.float64)\n",
    "    net2 = net2.to(torch.float64)\n",
    "    net3 = net3.to(torch.float64)\n",
    "\n",
    "\n",
    "print('==> Copying models initial parameters..')\n",
    "params1 = net1.named_parameters()\n",
    "params2 = net2.named_parameters()\n",
    "params3 = net3.named_parameters()\n",
    "\n",
    "\n",
    "dict_params2 = dict(params2)\n",
    "dict_params3 = dict(params3)\n",
    "\n",
    "for name, param in params1:\n",
    "    if name in dict_params2:\n",
    "        dict_params2[name].data.copy_(param.data)\n",
    "    if name in dict_params3:\n",
    "        dict_params3[name].data.copy_(param.data)\n",
    "\n",
    "nets = [net1, net2, net3]\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "for net in nets:\n",
    "    net.to(device)\n",
    "    print(net)\n",
    "\n",
    "optimizers = []\n",
    "lr = 0.01 * (npr.random(1)[0]/5.0+1) \n",
    "for net in nets:\n",
    "    optimizers.append(torch.optim.SGD(net.parameters(), lr=lr))\n",
    "    \n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                 \r"
     ]
    }
   ],
   "source": [
    "for net in nets:\n",
    "    net.train()\n",
    "\n",
    "t = tqdm(iter(train_loader), leave=False, total=len(train_loader))\n",
    "NstepMax = t.total \n",
    "\n",
    "## Difference between weights\n",
    "difference12 = np.zeros(NstepMax)\n",
    "difference23 = np.zeros(NstepMax)\n",
    "\n",
    "## Activation of the first layer before application of relu\n",
    "active1 = np.zeros(NstepMax)\n",
    "active2 = np.zeros(NstepMax)\n",
    "active3 = np.zeros(NstepMax)\n",
    "\n",
    "for batch_idx, (inputs, targets) in enumerate(t):\n",
    "    ## Forward path for each network\n",
    "    for i in range(len(nets)):\n",
    "        optimizers[i].zero_grad()\n",
    "        if doublePrecision:\n",
    "            inView = inputs.view(-1, 28 * 28).to(torch.float64)\n",
    "            outputs = nets[i](inView)\n",
    "            loss = criterion(outputs, targets).to(torch.float64)\n",
    "        else:\n",
    "            inView = inputs.view(-1, 28 * 28)\n",
    "            outputs = nets[i](inView)\n",
    "            loss = criterion(outputs, targets)\n",
    "        loss.backward()\n",
    "        \n",
    "        ## Keep track of activations\n",
    "        active1[batch_idx] =  torch.min(torch.abs(net1[0](inView))).detach().numpy()\n",
    "        active2[batch_idx] =  torch.min(torch.abs(net2[0](inView))).detach().numpy()\n",
    "        active3[batch_idx] =  torch.min(torch.abs(net3[0](inView))).detach().numpy()\n",
    "    \n",
    "    ## Keep track of weight differences\n",
    "    sum1 = 0\n",
    "    sum2 = 0\n",
    "    for j in [0, 2, 4]:\n",
    "        sum1 += torch.abs(net1[j].weight - net2[j].weight).sum()\n",
    "        sum1 += torch.abs(net1[j].bias - net2[j].bias).sum()\n",
    "        sum2 += torch.abs(net3[j].weight - net2[j].weight).sum()\n",
    "        sum2 += torch.abs(net3[j].bias - net2[j].bias).sum()\n",
    "                \n",
    "    difference12[batch_idx] = sum1\n",
    "    difference23[batch_idx] = sum2\n",
    "    \n",
    "    ## Perform an otptimization step for each network\n",
    "    for i in range(len(nets)):\n",
    "        optimizers[i].step()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plots wieght difference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQ0AAACwCAYAAAAVMTsXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAMTQAADE0B0s6tTgAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO3de1wU9foH8M8KCHkp88KKXOQiqyELqwiigoAe0QyVKFHLu0YeMutwjnj8aV5eknrE7KhZWCbkJfKGmZdKTUDLG2YgpCYpKCgXQUUuoiz7/P7gx/zcgGVWdmVxn/frta+XMzuz8+wIDzPfmXkeCRERGGNMpFbNHQBjrGXhpMEY0wonDcaYVjhpMMa0wkmDMaYVThqMMa1w0mCMaYWTBmNMK1onjZKSEmRkZOgjFsZYCyAqaYwYMQL37t1DWVkZ3N3dERQUhEWLFuk7NsaYARKVNAoKCtChQwccOnQIY8aMQWZmJr799lt9x8YYM0CikkZVVRUA4Pjx4xg2bBjMzMzQqhUPhzBmjET95ru6umLEiBE4cOAAhgwZgoqKCn3HxRgzUBIxT7lWVlbihx9+gLu7OxwcHHDz5k2kp6djxIgRTyNGxpgBEZU0wsPD8emnnzY671ljbm6OLl26NHcYjOnV7du38fDhQ9HLm4pZ6PTp03XmnTp1SnxULVSXLl2Qm5vb3GEwplc2NjZaLa9xTGPXrl0YO3YssrOzERoaKryGDx+ONm3aNCnQx2VmZmLgwIGQyWTw8vLCxYsX613uyy+/hLOzM5ycnBAWFgalUgkAyM7OhqmpKRQKhfC6evWqsN6ZM2egUCggk8kwdOhQ5OXl6Sx2xowOaZCamkqxsbFkZ2dHcXFxwishIYFKSko0raqVgIAAio2NJSKiXbt2kbe3d51lrl27RlZWVpSfn08qlYpGjRpFMTExRESUlZVFnTp1qvezVSoVOTk5UWJiIhERRUdH0/jx40XFZW1trf2XYayF0fbnXGPSqFVYWPhEwYhRUFBAL7zwAlVVVRFRzS+5VCqlrKwsteVWrVpF4eHhwvTBgwfJz8+PiDQnjbNnz5KLi4swff/+fbKwsKBHjx41GhsnDWYMtP051zimsXbtWrz33nuIjo6u9/1Vq1Y1+UgnJycH3bp1g6lpTSgSiQR2dna4ceMG7O3theVu3LiB7t27C9P29va4ceOGMH3//n14enqiuroawcHBWLBgAUxMTOqs1759e7Rv3x55eXmws7NTi2XNmjVYs2aNMF1WVtbk78eeLqr5Qwji0rdqJBKJzu6t0pg0LCwsAABt27bVycYaIpFI1KYb+g9/fLnHl7GyskJubi4sLS1x584djBs3Dh999BEiIyO1+vyIiAhEREQI09oOELHmo1KpUFhYiHv37nHCaICZmRns7OzQunXrJn2OxqTx9ttvAwAWL17cpI1oYmtri9zcXCiVSpiamoKIkJOTU+cowM7ODtnZ2cL09evXhWXMzc1haWkJAOjYsSOmT5+Or7/+GpGRkXXWKy0tRWlpKaysrPT2ndjTd/36dbRq1Qr29vYwMzNr7nAMDhGhuLgYN27cQI8ePZr0WaKOV0pKSjB79mz06tULL730EubMmYOSkpImbbiWpaUl+vTpg23btgEA9uzZA3t7e7VTEwB47bXXsHfvXhQUFICIEBMTg/HjxwMACgsLhVvdHz58iISEBPTp0wcA4OHhgcrKSiQlJQEANm7ciODgYP7BeoaoVCpUVlbC2toaFhYWMDEx4ddfXqampujUqROqqqqgUqmatL9FJY0pU6aAiLBjxw588803wjxd2bhxIzZu3AiZTIaVK1fiyy+/BADMnDkT3333HQDA0dERS5cuxaBBg+Dk5ARLS0vMmDEDAPDzzz+jT58+cHd3R9++fdG1a1csWLCg5gu2aoVt27bhvffeg0wmw8GDB/HRRx/pLHbW/GpPR/56GsrU1e6fpp6+ibojVC6XIz09vdF5zxobGxu+uasFqK6uxpUrVyCTyWBiYtLc4RishvaTtj/noo407OzsUFRUJEwXFRXBwcFBi3AZMz5z5syBvb09JBKJ3gtXRUVFwcnJCU5OTvjggw/0ui2NA6G1Vx/atWsnFN8BgIMHD2Lo0KF6DYyxlu71119HZGQkfHx89Lqd48ePIz4+HhcuXICpqSkGDRoEHx8fDB8+XC/b05g0ai+1uri4wMXFRZj/1ltv6SUYxnRl5lcpuF6snxIO3Tu1waYpno0uN3jwYFGfJ5PJEB8fDw8PDwBAbGws9u/fj4SEBERFRWH79u0wNzcHAOzbt0/tviMA2LFjB6ZOnSr8vk6fPh3x8fHNkzT0eamVMVZj6tSpiI2NFZJGXFwc/vWvf+Hu3btYvXo18vLy8Nxzz6GioqLeG7Ru3LgBPz8/Ydre3h67d+/WW7yinnJ93FtvvYUvvvhCH7EwpjNijgQMxZQpU9CnTx+sWbMGOTk5uHLlCl5++WVIJBI4Oztj4sSJCAwMxCuvvNLgDYcN3fioD6LGNB63e/duvPjiiwB0cxs5Y8bO2toaffv2xXfffYe0tDRMmjRJeKzi9OnTOHnyJJKSkuDt7Y34+Hj4+vqqra/pxkd90Hj1ZMOGDbh16xbatm0rvACo/Zsx1nTTp0/H5s2bsWXLFkydOhVAzd3LBQUF8PX1xQcffAAfHx/89ttvddYdO3YsvvrqK5SXl+Phw4fYvHmzcOOjXmh6mu3y5cvk7+9PH330EalUKiIisre31+qJuJaMn3JtGZRKJV28eJGUSmVzh6ImPDycrK2tycTEhKRSKTk5OTW4bGVlJXXs2JH69+8vzMvJyaH+/fuTq6sryeVyCgkJoXv37tW7/tKlS8nBwYEcHBxo/vz59S7T0H7S9ue80Zu7VCoVVq5cicOHDyMmJgYjR47EtWvX9JfFDAjf3NUy8M1d4ujq5q5GB0JbtWqF//mf/0FQUBCmTp3Kj4szZuREXz1xc3PDzz//zKXyGDNyGgdCHz16pHb55ujRo4iNjcW+ffv0HhhjzDBpTBpeXl4oLi4GAKxfvx6RkZEoLy/HihUrsGzZsqcSIGPMsGg8Pamurkbnzp0BAFu3bkVycjJefPFFVFZWwtPTU+8PxjDGDI/GI43q6mph4LN169bo0KEDgJoygE0t5MEYa5k0Jo3Zs2fj5ZdfxrFjxzBq1CjMmjULSUlJmDdvHvr16/e0YmSMGRCNpyfh4eGwtrbG4sWLcfHiRSiVSpw6dQpvvPEGli5d+rRiZKxFyszMxJQpU1BUVIQOHTogLi5O7WlxXYqKikJsbCwA4I033tDvmKNWt4IZGb4jtGUw1DtCxTQB04Xk5GRycXGhsrIyqqysJA8PD/rhhx/qLKerO0K1fsqVsRbh6/HA3Sz9fPaLDsAb32hcpLCwEOfPn8fhw4cB1BTGnj17NrKzs+sUzW5p9TR00z2liZrayzU9PR2DBw9Gr169IJfLERYWptYFWyKRwM3NTejzeuLEiafyvZjx0tQE7K9q62nUiouLw7Rp04R6GufPn0dqaipOnjwJqVRaZ/3GGonpXJOOi3Skqb1cr1y5QmlpaURUcwgWGhpKH374obAuACotLdU6Lj49aRkM8fTk3Llzau1AiYj69etHycnJdZbNzc2lLl260MOHD+nPP/+krl27UlVVFSmVSurXrx+FhIRQTEwM5eTk1LutoKAg2rlzpzB94MABCggIqLOcrk5PRB9pVFdXqz2zryu1h3ETJ04EUHMYl5WVVWdbu3fvxquvvgqpVAqJRIJZs2YhPj4eAODs7Aw3NzcAgImJCTw9PY3moTpmmB5vAgagwSZggHo9jbi4OKGehomJCU6fPo33338fhYWF8Pb2rvco2aDqadQ6ceIEunfvLtQ8TElJwaRJk3QSgNjDOLGHYOXl5di0aRNGjRqlNt/f3x/u7u6IiIhAeXl5vbGsWbMGNjY2wosfzmNPSmwTsFotqZ6GqKQRGRmJ5ORkdOrUCQDg6emJ8+fP6yyIpvZyrVVVVYVx48YhMDAQY8aMEeZfv34d586dw8mTJ3H79m3MnTu33s+PiIhAbm6u8GrXrt2TfB3GADTcBKw+Y8aMwZkzZ2BlZSVcli0pKUFISAjkcjnc3NxQVVVVb5Myf39/hIaGQi6X46WXXkJgYCBGjBiht+8l6uqJUqmEk5OT2rymNpGtpYterkBNwggNDYWVlRXWrl1bZ12gpuJYeHg4wsLCdBI7Y5r07NkTp06dErWsubm58JxXLRsbG5w+fVrU+osWLcKiRYu0jvFJiDrSsLCwQFlZmfCX/vfffxc6yjeVLnq5KpVKjB8/Hh07dsTnn3+udkRy9+5dVFTUlLJXqVTYsWOH0OeVMfYExIyW/vjjjzRw4ECSSqU0ZcoUkkqldOTIEa1GXDW5fPkyeXt7k7OzM3l4eFBGRgYREc2YMYP27dsnLPf555+Tk5MTOTg40IwZM+jRo0dERLRt2zYCQG5ubuTu7k7u7u4UHh5OREQnT54kuVxObm5u5OLiQhMnTqTi4mJRcfHVk5bBEK+eGKKnVu6vVlZWFn744QcQEQIDA5vcrr4l4HJ/LYNKpcIff/wBZ2dnYUCd1aVUKpGZmYmePXuq9U/Rebk/oObKhZWVFf7+978DAB48eICcnBzY2tpqGTZjuteqVStYWFjg5s2bkEqlMDMza+6QDA4Robi4GGZmZvU2XNKGqKTx+uuv4/jx43XmnTlzpkkbZ0xXunfvjsLCQmRnZ+u9WVBLZWZmppP7N0QljUePHqkNfD733HNqt2kz1txatWqFrl27QiqVgog4cfyFRCJp8hFGLVFJQyKRoLCwEJaWlgAgXMFgzNBIJJI69/0w3RKVNObMmQMfHx9MnjwZALBlyxYsXLhQr4ExxgyTqKQxbdo0ODg44NChQwBqnjb9az9JxphxEH3J1RjxJVdmDPRyyTU7Oxv/+c9/cPXqVeGpPQA4duyY9hEyxlo0UUkjNDQUQ4cOxezZs7lXJmNGTlTSqKysxIoVK/QdC2OsBRB14dbV1ZXP7RljAEQeady5cwdubm7w8fFRu8lr586deguMMWaYRCWNCRMmYMKECfqOhTHWAohKGrXVgmoL5TDGjJeoMY2LFy9CoVDAwcEBAPDrr79i3rx5eg2MMWaYRCWNd955B5988onQQb5v3744ePCgXgNjjBkmUUmjtLQUPj4+wrREIuGaBYwZKVFJw9TUFFVVVcLTg7m5uTp7zJYx1rKI+s2fPXs2Xn31VRQVFWHJkiUYPHhwg20AnkRT2zICwIEDB9CrVy/06NEDr732mlrPkjNnzkChUEAmk2Ho0KHIy8vTWeyMGR2xxUR/+eUXioyMpLlz59Lx48e1KkTamKa2ZSwtLSVLS0u6dOkSERG988479O9//5uIiFQqFTk5OVFiYiIREUVHR9P48eNFxcWFhZkx0Hlh4erqaigUCqSnp+slaRUWFkImk6GoqEjoe2JlZYXTp0+rtTGIjo5GdnY2NmzYAAA4dOgQVq1ahaSkJOzatQtxcXHC4OzFixcxcuRIZGdnIyUlBVOnTsXvv/8OoGZ8xtLSEvfv3290XKaxp/9SV41Ah0q+U5YZtnsWNlBE/tDg+9o+5dro6YmJiQlsbGzw4MED0R+qDV20ZazvvZs3b0KlUtV5r3379mjfvn29pyjclpGxxom6U0smk8HX1xehoaFqrQrDw8N1EoQu2jJqKvEm9vMjIiIQEREhTNvY2DT4mQA0Zm/GnlWiksb9+/chl8tx6dIlYZ6u6jDqoi2jnZ2dWm2P7OxsWFtbo1WrVnXWKy0tRWlpKaysrHQSP2NGR9eDKk/Cz89PbSC0f//+dZa5evVqnYHQzz77jIiI7t+/T126dFEbCJ03bx4REVVXV5Ojo6PaQOi4ceNExcUDocwYaPtzLipplJSU0HvvvUdjxowhIqLff/+dvv76a+2ja0BT2zISEe3bt4969uxJTk5OFBwcTCUlJcJ7J0+eJDc3N3J2diZ/f3/Kzc0VFRcnDWYM9NKW8Y033oCLiwu++eYbZGRk4MGDBxgwYABSU1OfxsFQs+EaocwY6PzqCQBcvnwZCxcuFC5RPvfcc9z3hDEjJSpptG7dWm36wYMHnDQYM1KikkZAQACWL1+Ohw8fIikpCePGjUNwcLC+Y2OMGSBRSWPZsmWQSCRo3749IiMj4eXlhUWLFuk7NsaYAdI4EPruu+9i/fr12LZtGyZOnPg04zIIPBDKjIFOB0JPnDgBoOb2asYYAxq5I9TGxgZyuRxZWVnw8vKq8/7Zs2f1FhhjzDBpTBp79+7F+fPnMXHiRERHRz+tmBhjBkxj0oiIiMD69esxefJk+Pn5Pa2YGGMGTNSYxt69e59KMIwxw6fxSMPW1pbHNBhjajQmjYSEBB7TYIyp0Zg0zMzM0L9/f3z33Xd46aWXnlZMjDEDpjFp7Nq1C2PHjkViYiISExPrvK+ryl2MsZZDY9LIyMjA2LFjkZKSUuc9XVXuYoy1LKLqaRgrvo2cGQNtf841Hml8+umnGlfm0xPGjI/GpFF7WlJUVITk5GQMHToUAPDTTz9h2LBhnDQYM0Iak0ZsbCwAIDg4GGlpaXBwcABQU+07MjJS/9ExxgyOqHoa2dnZQsIAapoRXblyRScBVFRUYMKECejRowdkMhkSEhIaXLahnqyVlZUIDg6GTCaDQqHAiBEj1NoW+Pv7w9HREQqFAgqFAh9//LFOYmfMGIlKGp07d8ayZcuQl5eHvLw8REVFoXPnzjoJYPXq1TA3N8eff/6JH3/8EeHh4bh7926d5YgIb775Jv773//iypUrePnll9UaG4WFheGPP/5AamoqgoKCEBYWprb+unXrkJqaitTUVPzjH//QSeyMGSNRSWPLli1IS0uDq6sr5HI5Lly4gC1btugkgB07duCdd94BADg4OGDw4MHYt29fneXOnTsHc3Nz+Pv7AwDefvttfPvtt6iqqoKFhQVGjhwpXAb29vbGtWvXdBIfY0ydqKTRrVs37N69G8XFxSgqKsLOnTvRrVs3nQSgqUerpuU09WRdt24dRo0apTZv7ty5kMvlGDduXIMJhXu5MtY4UW0Zm8LX11etnePjfvvtNwCae7Q+TkxP1uXLlyMzMxMxMTHCvK1bt8LW1hZEhA0bNiAoKAgXL16ss662vVwZM0aijjSa4sSJEygqKqr3ZWtrq7FH6+PE9GRdvXo1EhIS8P3336NNmzbCfFtbWwA1SWf27Nm4du0aiouLdf9lGTMCek8ajRk7diw2bNgAAMjKykJycjJGjx5dZzkPDw9UVlYiKSkJALBx40YEBwcLDZzWrFmD+Ph4HDlyBB06dBDWUyqVKCgoEKb37NkDqVSKTp066fFbMfYME9O7cdmyZaLmPYmysjIKDQ0lJycncnZ2pl27dgnvffbZZ/TBBx8I0w31ZM3JySEA5OjoSO7u7uTu7k5eXl7C53t4eJCrqyu5ubnRkCFDKDU1VVRs3MuVGQO99HLt27cvzp8/3+i8Zw0/e8KMgU6fPTly5AgOHz6MW7duqd0BWlJS8uQRMsZaNI1jGq1bt0a7du0gkUjQtm1b4dWrVy+Nd24yxp5dok5P0tLS4O7u/jTiMSh8esKMgU5PT2r17t0bO3bswNWrV6FUKoX53M+VMeMjKmmMHz8e+fn58PLygomJib5jYowZMFFJIz09HZcvX+YSf4wxcTd32dnZoaqqSt+xMMZaAFHl/mQyGYYMGYKQkBBYWFgI73PlLsaMj6hyfwDg7OyM9PR0YZpPVRgzTqLK/THGWC1RA6H1VSV/4YUX4OXlBWdnZ50HxRgzXKKSxvfff4/jx48L1ciPHTsGHx8fzJ8/H0uWLMH06dP1GiRjzHCIfjQ+IyMDCQkJSEhIQHp6Otq2bYuzZ89izZo1+oyPMWZgRFcjry1kA9QUtbly5Qq6du0KU1O9F/9ijBkQUUlDKpVi+fLlyMvLQ35+PlasWIGOHTuiurqar6IwZmREVyM/f/48XF1d4erqil9//RVfffUVqqqqdFaVnDHWMnADaA34KVdmDHT6lOsvv/yCQYMG4dChQ/W+P3LkSO2iY4y1eBqTRlxcHAYNGoTo6Og670kkEp0kjYqKCsyYMQMpKSlo1aoVVq5ciZCQkHqXPXPmDN5++21UVFTA1tYW27ZtE6qR29vbw8LCQrjNff78+Rg3bhwAoLCwEJMnT8bVq1dhbm6OmJgY+Pj4NDl2xoySzquUamnp0qU0ZcoUIiK6du0aSaVSunPnTp3lVCoVOTk5UWJiIhERRUdH0/jx44X3u3fvTunp6fVuY9q0abR48WIiIjp79izZ2dlRVVVVo7FxYWFmDLT9ORc1EFpdXY21a9fi3XffBQBcvXoVx44d00nS0kVbxsbs3LlT2IanpyekUil+/vlnncTPmLERlTTeffddZGRk4OjRowCATp06Yd68eToJQJdtGd98803I5XLMnDkTt2/fBgAUFxdDpVKhS5cujW6D2zIy1jhRSePkyZP44osvhPGCDh064NGjR6I24Ovri86dO9f7ysnJAaCbtozHjx9HWloazp8/j06dOmHKlCmi1ntcREQEcnNzhVe7du1EfUfGjImo2zkfr6EB1JyuqFQqURs4ceKExvdr2y3WHglcv3693gHWxtoy1rZyNDMzw/vvvw+ZTAYAQie127dvq22jvtaPjLHGiTrScHNzw/bt20FEyM7ORnh4OAYPHqyTAHTRlrG8vBz37t0Tlo2Pj0efPn3q3UZKSgry8/P56gljT0rMaGlpaSmFhYWRpaUlWVpa0syZM6msrEzrUdr66KIt49WrV0mhUJBcLidXV1caPXo0ZWVlCevl5+fTsGHDqEePHuTi4kJJSUmiYuOrJ8wY6LQtY1RUFAICAtC/f3+jfDCN7whlxkCnd4RmZ2djypQpyM/Px6BBgxAQEIAhQ4bAw8PDKFoZ3L59GzY2NhqXKSsrazEDpi0l1pYSJ/BsxFp7pVEsUc+e5OTkICkpCUlJSUhMTERRURF8fX1x8OBBrTb2LGpJRyMtJdaWEidgnLGKOuewtbXFhAkT4ODgAHt7e2zbtg2pqalN3jhjrOXRmDROnTqFxMREJCYm4saNG/D09MTgwYOxf/9+4ZImY8y4mCxZsmRJQ292794dFRUVmD9/PtauXYuQkBB4eHgI9z6wGgMGDGjuEERrKbG2lDgB44tV45hGUlISkpOTkZSUhJs3b6J///7w8/ODv78/evTo0eSNM8ZaHtFFeB49eoRTp04hKSkJ8fHxKCsrazEDQIwx3RE1EHrr1i0kJiYiKSkJx44dQ2FhIQYNGqTv2BhjBkjjkUZYWBiSkpJw69YteHt7IyAgAAEBAfDy8jLKm70YY408e2JtbY1Nmzbhzp07OHr0KBYsWICBAwdywgCQmZmJgQMHQiaTwcvLCxcvXmzukAAAlZWVCA4Ohkwmg0KhwIgRI4QH/fz9/eHo6AiFQgGFQoGPP/64eYNFTZmCXr16CTHt2LEDQE21tREjRsDZ2Rmurq7NXv/k3r17QowKhQIymQympqa4c+dOs+/XOXPmwN7eHhKJBBkZGcJ8TfuwoqICEyZMQI8ePSCTyZCQkCB+gzq9id2IBAQEUGxsLBER7dq1i7y9vZs3oP/z4MEDOnjwIKlUKiIiWr9+PQ0bNoyIiPz8/Gj//v3NGV4dDVVce9Jqa09LdHQ0BQUFEVHz79fk5GTKycmpsy817UOxFfPqw0njCRQUFNALL7wg/AeoVCqSSqVqD8kZipSUFHJyciKi5v/hrk9DSaNt27ZUWFgoTHt6egqlHg2Bi4sL7d27l4gMZ7/+dV9q2ocuLi509uxZ4b2xY8cKfwQbI7otI/t/OTk56Natm3CaJpFIYGdnV281sOa2bt06jBo1SpieO3cu5HI5xo0bh2vXrjVjZP/vrxXXtKm21hxOnTqF4uJiBAUFCfMMbb82tg/FVsyrDyeNJyS2GlhzWr58OTIzM/Hhhx8CALZu3YpLly7hwoUL8PX1Vfuhby4NVVwz5P27efNmTJ48WfijYYj7FWh8H4qtmFdHk4+JjFBBQQE9//zzBn16Eh0dTR4eHnT37t0GlzE3N6eioqKnGJVmt27donbt2hERUZs2bQzy9KSsrIzat29Ply5danCZ5tqvfz090bQP+fTkKbO0tESfPn2wbds2AMCePXtgb28Pe3v75g3s/6xZswbx8fE4cuQIOnToAABQKpUoKCgQltmzZw+kUmmzPhKgqeKaoVZb27VrF9zc3NCrVy8Ahrlfa2nah2Ir5tVLJynOCF2+fJm8vb3J2dmZPDw8KCMjo7lDIiKinJwcAkCOjo7k7u5O7u7u5OXlRWVlZeTh4UGurq7k5uZGQ4YModTU1GaNVVPFtSettqZvPj4+tHnzZmHaEPZreHg4WVtbk4mJCUmlUmHgW9M+1FQxrzHcy5UxphU+PWGMaYWTBmNMK5w0GGNa4aTBGNMKJw3GmFY4aTzD7O3thace4+LicOXKFZ1v4969e1i1apXavJkzZzbajtNQ1fd9mDpOGkbiSZOGSqXS2Le3vl+yTZs2wdfXV+ttGYKmJA2lUqnjaAwTJw0jsGnTJpw7dw5z5syBQqHAoUOHAACrV6+Gl5cX+vbti5EjRyInJwcAsGTJEkyaNAkhISFQKBTIy8vD3Llz4enpCYVCAT8/P2RmZgIAZs2aJdSa6NevH4Cauh0HDhwAABQUFODVV1+FXC6Hq6srPv/8cyEue3t7LF26FAMHDoSDgwOioqIa/A6xsbFQKBRwd3dHv379hBohW7duhVwuh5ubG1555RXcvHkTQE2SfP3114X1Dxw4AH9/fwA1tW8VCgXCw8Ph7u6O3r1749y5cw1+n/z8fISGhsLLywtubm5YtGiR2nf48MMPERAQIDw388zT8c1pzIA8/izCXx/f3r59O7311lukVCqJiGjLli00evRoIiJavHgxWVtbU0FBgbD87du3hX/Hx8fTK6+8QkREWVlZ1KlTJ7XtPr6t0NBQ+ve//01ENc/s2NjY0JkzZ4T43n//fSIiKiwspOeff17oz/u4xMREcnJyolu3bhERUXl5OZWXl1N6esq99t8AAALzSURBVDpJpVJhnaioKBo5ciQREcXGxtJrr70mfMb+/fvJz89P+DxTU1NKSUkhopqewYGBgQ1+n8DAQEpOTiYioqqqKho+fDglJCQI3yEsLEyoX2IMuASXkfr2229x7tw5eHh4AACqq6vVWm0GBQXB0tJSmD58+DDWr1+P0tJSqFQq3L9/X9R2jh49irS0NAA1z+yEhITgp59+gpeXF4Cax+IBoEuXLnB0dERWVhasra3VPuPgwYOYPHkyrKysAABt2rQBACQmJiIoKEhYPjw8HFFRUaKe2OzZs6dwJDFgwACsXr263uXKy8tx7NgxtedLysrKcPnyZWF62rRpdZ4ofZZx0jBSRISFCxdi+vTp9b7/eM/PGzduYM6cOTh79iwcHR1x4cIFDBkyRPS2/voL9fi0hYWF8G8TExOtxgWISO2zHv+3qakpqqurhenKykq1dcVuV6VSQSKRICUlBWZmZvUu01J6ueoKj2kYieeffx4lJSXC9OjRo/Hpp5/izp07AICqqir89ttv9a5bUlKC1q1bo2vXriAifPLJJ2qfW1FR0eAv3d/+9jdhHOP27dvYu3evVgkHAEaNGoUtW7YgPz8fQE19y4qKCgwdOhSHDh0S5sfExGDo0KGQSCRwcnJCWloaKisroVQq8fXXX4va1l+/T/v27eHr64uVK1cKy9y6dcuo23fwkYaRCAsLwz//+U9ER0dj+fLlmDRpEoqLi+Hv7w+JRAKlUokZM2YIj6Y/Ti6XY+zYsejduzfs7OwwbNgw4b2OHTsKlbfatm0rDCjWWrduHWbNmgU3NzeoVCosWLBAODURa/DgwVi4cCECAwMhkUjQunVr7N69G71798aKFSsQGBgIoKbncG2CGjBgAIYPHw5XV1fY29ujT58+QoLUpL7vs337dkREREAulwOoObKIiYmBjY2NVt/jWcFPuTLGtMKnJ4wxrXDSYIxphZMGY0wrnDQYY1rhpMEY0wonDcaYVjhpMMa0wkmDMaaV/wXb+cOprSF23gAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 240x160 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "\n",
    "plt.figure(num=None, figsize=(3, 2), dpi=80, facecolor='w', edgecolor='k')\n",
    "    \n",
    "plt.plot(difference12[0:100], label = \"1 vs 0\")\n",
    "plt.plot(difference23[0:100], label = \"0 vs 0\")\n",
    "plt.xlabel(\"Iteration counter\")\n",
    "plt.ylabel(\"Weight difference \"+bitsString)\n",
    "plt.legend()\n",
    "\n",
    "plt.savefig('diffRelu'+bitsString+'.pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot minimal magnitude before activation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No handles with labels found to put in legend.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAACyCAYAAACNzhl4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAMTQAADE0B0s6tTgAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO19eXhU5dn+fWafyTJZSEJICAOEgEBCQAUXEJFFCogWCbW1KlULrtQfKl8/raKtWCpKvVRacMNPsVhxaevWaqmAVRQQZUd2khCy7zOZ/f39cc77zjmz5Uwyk5kk576uXJCZk3OeOXOe9372lyOEEChQoKDfQRVvARQoUBAfKMqvQEE/haL8ChT0UyjKr0BBP4Wi/AoU9FMoyq9AQT+FovwKFPRTKMqvQEE/hSzl/+CDD9Da2goAePrpp7Fw4UIcPHgwpoIpUKAgtpCl/A8//DBSU1Oxb98+bNq0CTNnzsSdd94Za9kUKFAQQ8hSfo1GAwD49NNPsWTJEixduhRWqzWmgilQoCC2kKX8Ho8HX3/9Nd59911MmzYNAOByuWIqmAIFCmILWcr/xBNP4I477sDkyZNxwQUX4IcffsCIESNiLZsCBQpiCE5OV9/+/ftRUlLS6WsKFCjoPZDF/IsXL5b1mgIFCnoPNOHerK+vR21tLex2O44cOQJqJLS0tCgBPwUKejnCKv+bb76JZ599FlVVVZgzZw573Ww2Y8WKFTEXLlrQ6/XIysqKtxgKFMQUdXV1cDgcso+X5fP/7ne/wyOPPNItweKJ/Px8VFZWxlsMBQpiikif87DK73A4oNfrYbPZgr5vMpkilzAOUJRfQX9ApM95WLP/0ksvxd69e5GcnAyO4yBeJziOg8fj6bqkPYAtW7Zgy5Yt6OjoiLcoChQkHGSZ/b0dCvMnFs41d8Dl9sIyICneovQpRJX5xTh37hy++OILcByHKVOmYNCgQV0SUIGC+9/+Hk1WF/71/66ItygJDa/XC39u5jgOKlV0mnFlKf9bb72Fe++9F5MnTwYA3HvvvXjhhRewaNGiqAihoH+hqtmONrtSHh4KTqcT5eXlIUvotVotCgoKoNPpunUdWcr/2GOPYdeuXRg6dCgA4MyZM5g9e3bCK7/i8ycmWu0udDgTO14UT5SXlyMlJQWZmZngOE7yHiEEDQ0NKC8vR2FhYbeuI8t+GDBgAFN8ALBYLBgwYEC3LtwTKCsrw9tvvw2j0RhvURQI8HoJWjtccLi98Hj7fLgpYni9XrhcLmRmZkKj0UCtVkt+NBoNMjMz4XK54PV6u3WtsMpvs9lgs9kwc+ZMPPHEE6iursb58+exatUqXHfddd26cH9HZZMNLR39z/S1Ot2gOm9zuuMrTAKC+vj+jC8Gfa+7sfqwZr9/iu/RRx+VCPDAAw906+L9FV4vwTXP/xezRg/EHxb2r+Yo8YLX4fQgxaCNozT9G2GVv7tmRX9EXZsDmUk6qFShV+4mmxNNNhfq2uWXYvYViJXfpvj9cUXEOYNDhw7FQo6YYMuWLVi0aFGPBfyO17Rh0pP/xscHz4c9jiq90504i2uH09NtM1IOWjt8pr6i/IGQY9LLcQ3kQJbPL/6ZN28eOjo6Qpb8JhKiFfA722DF0//6oVNlPVTVCi8Bjte0hz2urk1Qfk9iKH9Nqx0Tn/w3XvvqTMyvJWX+3uvzdzg9ePj9A6husUf1vCqVClqtFg0NDXC73fB4PJIft9uNhoYGaLXabuf7I/L5KZKSknpFeW+08I/vq/DC5ydw4ZB0TBuVHfK4ikZ+QWy2OcOejyl/gjD/B/uq0GZ340x97Nu0W+2dm/2EEPxjXxVmjR4Io04dc5m6gj1nG/HmN+UYkmnCkiuGR/XcBQUFKC8vR2NjY9D3aZ6/uwir/IsXL4ZKpcIf//hHpKSkAACGDh2K06dPd/vCvQkuITy960xjeOVv4pW/yRY+ik+V35UgzP/RAd5N6QlLpFWGz//N6Ub86q3v8bvrxuKmS4ZEXYZTde148uMjuOHiAswYndOlc9CFu7ol+nEbnU6HwsLCmFf4hT3Lq6++iuuuuw7Tp0/HJ598wi7e3+AWlGL36eArMUVlEx9baOpFzF/ZZMN35c0AAKe7J3x+UbTfFdzsr2nlTemaKJvUAJ9pWfHOfvz7SC1uf30P/ve9A106D1P+1tjFk1QqVUCeP1qKD8gI+M2bNw+ffPIJXn/9dSxevBhud+/107oKWoyyv7IFdldoV8fH/J0of3viMP9H+33ByZ5gfjnR/oZ2/v41WMPfx67g7T0V2HO2CQsm5GFcvhmbd5V3qd6C3qto+/w9CVnLSGZmJjZv3oy5c+fimmuuibVMCQe3oPxOjxf7KpqDH+PxoqqZfxCarNKHaV9Fs8TXTSTm/+xwDZL1GnAc4HTHPoYjUX5HCOW38ven0Rpdk9rl8WL1P48iJ1WPx+ePwSXDMnmZOnHTgoF+dzWtvTddG5ENUVZWhhdeeCFWsiQs3CJG3H0muOl/vsXOLARxwK+uzYEFf/4KL+04JXkNAJye2JjZHi/BQ+8fwI5jdZ0eW9fuQEGGCTq1qkcWo1Z756k+yvyNUWb+ZpsLzTYXZo0eiBSDFmYTX2DU3BH5dVzCd1fTaoe3l5Yph1X+devWoba2FgBw+vRpTJo0CTqdDuPHj8eRI0d6RMDuIFp5frfoy911pinoMdTfBwCr0yMKCPGLAvVjAaCWMX9opt32Qy3G//bTLpmVp+ra8ZdvyvHC5yc6Pdbm9MCoU0OnUfW82R/C56fmfrTNftpJmGrk49xpRr4rrrlLzM9/d24vQX0XLZRmmxNXPbMNW4/UBLx38FwLpjz1H5yoDZ827g46Vf7sbD66/cADD+D2229Ha2srHnnkEdx1110xEypaiFaenzL6yJwUfF/eFLQAg/r7eWn8tSj7UxO23cE/6A63hymAKwzzH6pqRZPNhWM1bRHLe6Sa/5tvzzZ1atLanR6YdGroNT3E/B0umI0844bq7Gtop2Z/tJWf/w5oSXEaY/7IlV/83XXV7z9W045TdVbsChJI/se+KlQ0duDrUw1dOrcchFV+cXnv6dOn8ctf/hJGoxELFixAc3Nw37cvgn7RJflmtNrdOB/ky64UcvzFeWYAQKNNarq2C/5tfbvvgQ7HtFQxuhKM+qGa31HZ4yX44kR407/D5YFBq4a2h8z+lg4Xcs0GAIA1hM9P71mzzSVxuboLGndJMVDm1zKZIoX4u+uq8tOYRrAA8Zcn6gEAVc0xzCaEe7OoqAjvvfceAGDkyJE4evQoL1BVVcwESkR4hEVwrKDYR863BhxTIZj9xfn8MTTox5RfePCov8+fl4Rsa+0QsgpdYaUfqtugFnoL/nO0NuRxTrcXbi+BUcub/Y4e8fldyE7llT9Uqq9BtEB2VjMRCfyZn/r8LZ1kZ4JBfK+qW7uq/Pxn83c7mqxOHBaesXPxUv5169bhySefxBVXXAGHw4FJkybhyiuvxNSpU/HUU0/FTKhEA/X5x+alAgCOVgea4pVNNqSbtMhP9zf7KfPzDx5Vftr3EyrdR1OKXXkwj5xvw4jsZIwamILtP9SFDEjRBcakU/MBP5ksSwjB6zvP4HxLZA+mw+2B3eVFukkLg1YVNODncHvQ5vAtCmLT/0y9NWyqtTO0+TO/qes+vyuKzO+/wH99qgHUszzXFCflHzx4MPbs2YOVK1di2rRpWLlyJe69917s27cPM2fOjJlQiQa3h4DjgFEDU8FxIZi/sQODM0zsgaKM1dhOmV+q/AMF9gulcIz5I3wwW+0unGvuwKiBKZg2KhsNVicOnGsJfg1B+QwC88utO/ihpg2P/v0QNn19NuC9o9WtKF75LxyqCrwmNa9TDVok6TRBlZ8qu07NP5pi/3/WH3fg1S+7Xl1KmT/Vz+wPZV25PF7c9Mo3+ORAYKOW2EXqqvI3MPdGusB/eZI3+TOSdDFlflljvKZPn47p06fHTIhEh9tLoFFxSNJrMCTDFMD8Xi9BbZsdxflmZDDllzJ/mx/zD0ozoqrFHtLPtnfR7D8myDZyYCqGZ/HTcY/XtmPc4LSAYyXMH0HAr1bIbYtdGIofqtvQ5nDj4LkWjBlklrxHO/rMRi2MOnXQxh5q8g/LSsLR6jZ2/5psTjg93m75wK1+Zr9Jp4ZWzYVcYGta7fjieD0GZ5jwo+JcyXsS5u+i2d8kim2I8dXJBgwyG1CSn4ZPD1fD5fFCq45eZR9F9M/YB+HxepkPPWpgKk7VtUvMz+YOF7wEGJCsZxFk+sU2iqL9hBDUtfMPSp7gHoRi264G/OjCNGpgCgYJmYfzIRSGKp9Rq44oz+8rwgl0SdweWusQKDdjfqMGJp06KPNTZS/KSZFcg96nUIVBcuBv9nMcB7NRi5YQeX4akAx2X+hrOan6KDC/y7cPps2FU3VWTBqWibx0I7wkdlWEivLLgNtLoBVqqkflpsBLgD9tO4mpaz7Hidp2ZpoOSNYhPcnP7Be+YEL4nHpdmwMqDsihZn9I5udfj7T67Aeq/LkpGChE1atCPDx0ATNGyPzhym+pkgZbtGi0nWd+TdBUH72XRTnJkmu4hL4DazfagP0DflSWUMxPrxVO+QsyTKhutQekfx0yqiXps+H0eJkVRs38IZkmljaOlenf65T/d7/7HZYtW4b777+/x/oM3B4CtdrH/ADw3NbjONtgw3flTSx9l5mkQxIzJQMVpN3hRqPViXSTDgYt36oaSuF80f7IAn4HzrUg1aDBwFQDMpN00GlUIQNzlHmNNM8v0+evC5OHpx2QwdwV2tRjNmqRFIL56TkLsynzS2cfhEoPykGb3QUVBySJ2oTTTLqQrpUtHPML8gzOMMHm9EgqFz/cX4Uxj/4L5Q3hZ140WQOzGvS7GmQ2MuswVkE/2cp//vx5fPXVV9ixYwf7kYNly5bBYrGA4zgcPHhQ8t7x48dx2WWXoaioCBMnTsThw4fDnuvDDz/EoUOHkJSUhJycHGg0svccCQlCCOwuD4vGB4NH8PkB4IJc/qGkzY21bQ72wGYm68FxHNJMOt5HdXsZ2wA88zTZXEgzaaETFpNQCmfvQsDvXHMHvq9oxpSiLHAcB47jkGs24HxzcOanzEt9fpeHyCpVZczfHkT53WGYXxTwM4Xw+elCWpjNM7+/2d9d5uf7GHydqWlGLVpEZrcY9JkI9h053V6oOJ75AT7bQ/HX3RVwewnONoaej0AIkRADJQtqpeWmGRjzxyrXL0t7Vq1ahTVr1mDYsGFQq/lVk+M47Nq1q9O/XbhwIVasWME2/BBj6dKlWLJkCRYvXox33nkHt912G3bu3Inq6mrccMMNkmNLS0uRm5uLkSNH4vHHH8cjjzyC7du3Y+rUqXI+Qkh0uDwY/ei/sGB8Htb+pDToMS6vFxrB7C/IMOH/zSjC0KwkLNv8HWpb7cyHzEzmTf50kxZNNldA8Ua7w40mqxPDspKg0/Dn64z5I/H5P9jH119cO863m1Ku2YDDVYHZCfE1jEKRD8A/6AZV+AEa1DRvd7jhcHug1/iOd3uDuytujxdfC5VsqYLZ7/KQgGAWPXd2Kh8/oQsMvU/d8/ndAQNDzSYtnB4v7C5vwOAQWxiz3+XxQqdRwZLJB1XPNtgwZpAZTVYnvjrJV+WF25vA5vRIagXoIk/jM7lmIzIFFzJWZr8s5X/11Vdx4sSJLs3qv+KK4Fsy1dbWYu/evfj0008BANdffz3uuecenDlzBhaLBdu2bQv4m7feegs1NXwddGZmJlpbgz/UkcCoVUPFQZJb9ofHS1jAj+M4/GrGCHi8BPe99R1qWh0wCxH+Acl6AEC6SYcfatrYg5udokdtmwOtHfyCkG5KZw98qBJf+uA43F7YhSq8zvD376uQatBg6sgs9togsxFfn2qE1eFGkl76dUtSfWLl7+Ra4irFRqsTuWZf+TT9POJFy+Xx4pZXd+Grkw2YNDQDowamwCRcw+b0wGxUSc6nU6uQotcgI0kXZeZ3sYWagtX3dzhh1EnLwK3O8Ga/Vq1i+w2eaeBZ/rPDNaxwyx4mhkI/V67ZgPMtdp/yC8w/KM0Ao1YNk04dX59/4MCBUd+ko6KiAoMGDWKmO8dxbHxRKCxYsAD79+/H/fffj/3792P27NlBj1u7di3y8/PZT3t76OYIjuOQrNfAGkb53R4CjVo6xESt4jAgWY+aNjtjK7pSp5t0aOlwoV54fUgmbxpWNXfAS/j8bWfML8kmyDD9j9e04cj5VswpzpUwcW4aH/QL5vf7Un2aTuURo0E0ddjf9KdKKo5VfHu2CV+dbMD8cYPwxm2ToFGrGMv6s2O91YmMJB04jsOAJH1gtL8bQz/b7G6k+jE/q+8Pco9ttB8jiNnvchPoNSpYhO+WjkATD28NV5BEP9cwIR1LrcSq5g6YjVqYdLx7kpdmjJnPL4v5r776atx///248cYbYTAY2OujR4/u1sWDbUUUDjqdDq+88kqn512+fDmWL1/Ofs/Pzw97fIpBG9bnd3u9zOcXIyfVgNpWB3JSnFBxvoqx9CQtCOEHfwJAQUYSdp9pYs0/6Uk6EfOHj/YDvCLRyH0ofCZ0hs0rkW6gSlm5qtnOgmgULOAnFPmEk4eCEIJ6ka/qH/GnqT6x2U87HueW5LLrJOl55fdn8karg7lPGUk67D7rhMdLWPtzuO+pM/Bmvx/zh1F+SgjBFkSHxwudWoU0kw5moxZnGmxotbvw5Yl6pOg1aHO44ZCh/MOzkvHliQZmKZ1vsbPeB4BPCe882QBCSNSnaMlS/o0bNwIAq/MHeMU9depUqD/pFIMHD0ZlZSXcbjc0Gg0IIaioqIjKYEIKuXv1Jes1rAIvGMSpPjGyU/Q4Wt2K+nYD0k065hpkpfBf3m6h/Zcyf0UjL0e6SQu9oATB6undHq8kyCSH+em5aUCSYpAf89ucbry79xyuKckNSPUBnTN/u8MNp9sr5MddAQM36OLRanczd4kyFw1gAby1AUiZ3+H2oLbVgaED+GBfRrIOhPDBMBpIdLq9XSp6sbs8cHq8AcpvZs09gcFLn9kfqMROtxda4Z5ZBiThbIMVu083wuUhmFOcjb9/XyVZwP3BmF9wG5qsTni9BNUtdkwe4bOyCzJM2PZDHWpaHZ0SQKSQdQdPnz4d8NMdxQeA7OxsjB8/Hps2bQIAvPvuu7BYLLBYLN06rxhyW3qTDRrZPr8Y2akGuDwEJ+raGVsBwIwL+DboTwQTkCp/udD5l26SMr/V4caJWl/VIPUVqR8uJ+hX3dIBnVqFjCTpzq1i5j/X3IHr/7wTj/ztIP723Tlpqk8tT/mpv8/y8AFmv896o0U155r5z037HgDe2gCkZvwXx+rhcHtx2XB+wg410dvsbsli2BXTP1iOHwhf388CfsHMfoH5AcCSaUJNqwPbheEpV4zgYy5yzP6hWfx9bO5wod7qgNPjZQs2wFsGAHCyLvp9/bKXzz179uAPf/gDnnrqKXz77beyL3D33XcjPz8flZWVmDFjhmRn0Q0bNmDDhg0oKirC6tWrZZn0kUDuMI9Omd9DoFUHM/v5AF+zzYXMJD17vTjPDEumiSkCTQfR0d4ZSTpfgM3txYbtJzH72S/Y+/ShyRbOL6fQ53yLHQPNhgDTcJCg/KfrrVi0fifrS2i0On0+v8jsD2aJHK5qZSlAXxEOb2H4m/2uIBbLueYOJOnUjGUBPr0ISGf3fyzU0M8VSmmZNeLxSs4r/pvvypuCVsARQvDG12dRK5Te+lf3UYSr72/vpMKPLuBDhIj/P/bxAddxg/myZnuYQh963woyTNCo+LoQmpIVB1BpTCBuyv/SSy9hwYIFOH/+PKqqqrBgwQK8/PLLsi6wbt06Zt5XV1fjxAnfdJmRI0di586dOHbsGPbs2YMxY8Z07VOEgGzm12vQ4fKE7B13i8p7xchO8a3QYubnOA7zS/OE/wP56bzy0y88PUnHTEaXx4vqVjvcXoJ/HqwG4DOFqe/nX+hzrKYNb++ukLxW3WoPahbSUtqPDpzHueYO3D55qHBO3zbZErPf7x7sLW/CnOe+YDEFyvwjBwpFOH7M7xbNgKAWS1WzHXnpRsnCZBIyD612Nz47XIOWDhc+O1yDcflmDBYWS73IFRErIPXFG9odWLRhJx58Z1/A5z5a3YZH/nYQb37DB5BDM3/nAb9wqT4ALOjXbHPhIksGjMylCW1F0QKfjCQdX2hkc/kKfIIxfwwm+shS/ueffx7ffvstnn32WTz77LPYs2cPnnvuuagLEy8kCw+iNYQ5yRf5BN4qyvyAL81HMV/ItaebdGxsFEW6SSdJrVEz9l+HeOWnzD/QTNuDpQ/mhu2nsOLd/awwxO7yoNnmkgSKKGihj8dLkGs24FczRrBzdjg94DheyXQhzH7KRtRvpxmMoQOSoFFxAcwvHv/d3OGC10twrrlD4u8DYKm+P31+Ar98fQ9mrt2ONocbc0t8DTTiuIjYnaBVfv88VA2Xh+DLE/VMLgrfQBChsYp29Bkj8fnDl/cys1/w2wHgYksGc2k6Y36NikOqQYM0kxbNHS42AFbM/ANTDTDp1DhZF/0NVWSb/VlZWZL/94b5/bLNfsEUDBVJdgVJ9QG++nzAl+ajKMxOxqShGRiZkwK9xsesAJBh0kGnESr83F7GwN+WN6GuzcHMcR/z+w178JsSRM3eUAEh+jDde9UIpAjVdc0dLthcHhi1anAcxywR/wedytLO2Ja/5oBkvZCHlyqdP/PXWx1wur2sVJWCmv1Hq9ugU6tYyfCPxvqUXxyEFJv9VCnp2HEvAT4RrCYKumDSsluf2S9l/hSDFhwXPK7CGnuCVfhJmN+n/BOHpsOg5V8P7/M7kC6kNNNNWt7sF5X2UqhUHIZlJcXE7JcV7S8sLMTDDz+Mu+++GxzH4cUXX8Tw4dHdoigWKCsrQ1lZWaepPsr8ofz+kAG/FB/bZ/oxPwD8360T2d+l6DVocDuhVnFIMWigEyolxcxPCPDvIzXM1KPn938wKZs1+RWG5KYGV/55JbkwaFUou4i/D3xJqxN6jZqxlJj5XR4vPF4Cg1aNDkHRmPILyp6ZrENGkg4NVic2bD+J6lY7Vl4zRqKkLTanKNJvksgkrqabXzoIZRfmo7bNwUx+sUwOt0eyKNkcfIPU16cacHlhJvacacJH+6sku/tQV4mWFPvMfukjr1ZxSDVoA8atA75FhpY9i3dedrp9yp9u0iLFoIHT7UVxXhpLCzvCRPubbC5GGGajDs22Zlbam2OWPkvDs5Jx8FwV2h1u9qxGA7KYf/369Th58iRKSkpQUlKCo0ePYv369VETIlaQy/wpjPmDB9ZC5fkzk/VsIo/Y56cwiMpmqXWRbtJCpeKgFZjf5SawuTxINWigUXH49FA1Y9tkvQYpek1AwI8uBnQRoLvGhGL+GyYW4OVbLmaymIVmFpvLzZRQ7PP/9oPDmPPcFwB8kXWqPA3tTnAcb70MSObbWZ/59Bje+baS/zwi87ylw8Wq08R+LABJteE14wZh0rBMXDNOWqOg14Zm/n8eqoaXAIsuGoxpI7PxzelGFtzj7w1lfum/qYZA5UkxaNAW5LsXlxKL2Z8QIlT4+ao+r5+Qj59NKoBOo4JKxUGnVoVlfnG1YbpJC7eXYMexOuSlGSVFWoDP7z8dZdNf1jKSnZ2Nt956K6oX7glEyvxtIZifr/ALXCdplV9tmwMDgih/sGukC6kln8/vQYfTjQEpemhVKpxpsEkCcWaTNiDgR5Xfn/kHmuVNKU4zalHV3AGNimPMrxcFII/XtuFUnRUeLwkw++vaHUg36aAR0oo0O0BDIv7Rfsr8+X5mP71uRpKOpfb8IbaOxMpndXiw/YdaaNUcpl+QAxXH4Z+HqvGvQ9W46VILAB/j0wEioQJ+9LVg3724AMnh9pU9e7wEhAA6kZI+Nl8arNZrVWF9fn5kOh0nphU+lxvPBukvoRH/U/XtbEZkNBBW+b/88ktcfvnl+Pjjj4O+P2fOnKgJEk+E8/kJIWySTzDkpBpQ2+aQpPqCXsNP+cW1/R0uD8xGLZL1Ghw538YYw6BV88EgEfMTQtjvjPmp2S+zCCTNpEWr3QWjVs2CluJUH2X7DpeHLUR0AGlDu4PVEohrCqhZ7vbwLpLHS9DS4WJK62/2ZwjtzwvG54Us2GEyuQJTfVXNduSkGpCs1+DKkVnQqDhsPVrLlD8U8/ub/fS1Gr9pPIQQScm32O2gnylY+pfCoFWHLPIhhP/OadCTBkN/M3c0pl8QuHForCL+YZX/tddew+WXX441a9YEvMdxXN9R/jA+P+1wDebzA3zE/8A5vhotHJiJl8Sv8uI0VofTg9xUI9JNOrTaXWwRMmrVSDfpcLLWyso7rU4PGygqbgahVogcpJn4yrm6dgcrQGIs6/ay69sc7gDmb7a5MFxot80SYhJ0+CetTEw1aNBqd6NFaGTSqjlJfATgzf5tD05jrBcMkjy/Wxrtr21zoCDDKNxbLSYNy8BXJxtgc7ph0mmYtdRm75z5Uw0atNldkhJah9vLvnsqAwWVRa8J7TUbtKqQXX12lxeE+IKeP51UgIuHZmB0bmrQ44cOSALHIeoR/7DK/9JLLwEAPv/886hetKcQSXkvEJz5XWyVD/5F33KZBaMHmQMaRkJdg7Kl1i/VZ9SpmVJS39WgVWOQ2YgOVz1aOlxIExqGKJpEzJ+Tog+5QPmDKpzHS9gDKI6sU8azOT0Sn58QgmbRphsLJuTB4yU4VtOGD/efZwuATqNCqkGD5g4XWjtcyDUbJcEyiqyU8IuVJM8vUr5WO19WfNGQdPbaVaNy8OWJBnx5ogEzR+ewhbHd4Ybb4w06yIMixaCFy0Mkpr1/o5eY+R0e/p7owpQYG7XqkGY/G5+moy6XOmDeoRgGrRoleeaAluPuQlbAb+LEibJeSzREUt4LBFd+2p4ZSrGmjMjC8plFncriC/gJPr/IpO1w8bvmZAhWwTkh32vUqtkcPtocI570Kmb+SOq+00SVdsECfjTFZXW6fWa/ww2r0wOPl7C/zzUbsWz6CGbV0H0ANCq+4eVMvRWn6qwYKsqDRwJdCOUvb7TBS6SLBy2p/s9RvhhJvEi2O4GfBFEAACAASURBVNxBB3lQUPnFm6n6TwySmP3u8IQA8AobKtovHpwqF3+/ZzKeLhsn+3g5kKX8/uOyPB5P2DbZ3oZwZj81sUP5/PKvwSuMv8/Pm5u8EtL3aL7XqFOx/Did5iKO/NNpQfXtDklhSGcQm9pGrUaQx5eeooEum9MjMfvpwiMu0wWkaUKaAks1alHbxteq33LZEHQF4lSfS6R8tFtS7EoMyUzC8KwkbD1SC6+XSOIkrR38+LRg6VjAp/zioB+9BwZRxoGCZjR04cx+jTpktL9D1E0ZT4RV/jVr1iArKwsHDx5EdnY2+zGbzZgyZUpPyRhzpAiKGYz5aclvsCKfiK7BfH6q/Pz5KEMZtWrWZEKj9watmqXIaMpMava7WKAqEuY3G33xCaOOfwSoid0qLEYAb/pSE7Xd7mYK5e+ni10YGhylC0RJvhnTRmbLlk0Mg1+qT69RQa3icFaYjZedKlXmyYUDUNvmQFVLhyRD0mp3obbNgayQyu9rIKKgn5suyE6PKO0ng/n12tCpPl9DVfRy9l1B2KsvWbIEZWVluPPOOyV5/dTUVKSnp4f5y8SAXJ+f9pYH6+zzMObv3qxTn8/PP2gcx+eCqTKLzX7K8gatGvlpJslr4mq/ZpuTLQpyI/2AP/PTIh/+X/FQSZvTF+13C3sTAGCTiyj8K/FMOg3ShWv8avqILleDUpkcHr68V6dRQQefkvrHDGiDzYnadkmkvcHqRKPViaxhnTG/2FXgP3e6SYfzLXZJwxONA4Vlfq065CQfqvyRmP2xQFjlN5vNMJvN+OSTT3pKnqhCbp5fo1bBqFWHNfvlBtNC4UdjB+J0vRWXDff1aus0PuU36jSM+emDZtSqkSqUn1Ilp+zLzwl0sgmx4hLTziBRfoF96IMsnjtoFUX7AV/cIcDsF8UL3B4CnZrDLy4fipEDU3DVqK6xvvi8Dhfv8+s1KmhUKqb84sYqAKw68KCwQxEdA3ZKKI0NFWAMyvwCEdDsjCTg59dyHQwGrRoeb+CMQsC3R2G0A3iRQhadnTx5Etdccw0KCgok5n9fQrIh+CgvOpmmu2Z/dqoBj80fI5mPJ94txqRTs91+KOg2WtkpehYEpIvFkMwk2F1etkmHZYA0jx4OaWKzXysN+Ik3xhRH+wFfc09aKOUXAnMatQqlg9Nw15WF3eoBES8qtIWWWmlAoDLTQiK6PRldDE52qvyBzE+bvOiC7IyQ+Y1h6vsThfllKf/tt9+On//850hJScHWrVtx3XXX4b777ou1bD2KZL0muM8vNKp0N+AXDDqNShL5TRcpv07wbwFI5rjR7jMaQd9X2SxpG5YDMfP7p/okzC+K9gM+5vf3+cUBP7eo7LW70Pu5E7zy05mPgc1UPubnZxbQOQonhOIY/1oDitRgAT/hWchgPn9gtL8zsx+AxHKi6BUBP4qWlhb85Cc/gUqlQnFxMTZs2IDPPvss1rL1KJL1mqAlnr5UX/T3NxGbg0atGikGDesVED8Yg9KMqG93sNZdjpOauIPMRlnTfSkMWjVTLP/GHvFGHDaHx8/s510Mf7NfqqQkavvK+ccStGqOLVaZSfqAkutkPR9roC4SLQI6UctnBzoz+1uDRPtp7KIrqT4geHMPG5neG5hfq+VvQEpKCs6ePQuHw4GzZwN3aE00yG3sAUIzP03raGPE/BQmnQYqFcfYn0a6Ad++fnTEc6pBy1jP4fayKr1IQNmbPoCUrf3TXcF8frHbIP4cDk/X5uuFgkbFgeOEVJ+wqCQJMYpQiizuCqTMT3v9/WMEFMHMftrU01Wz36Dxmf3Pbz2OrcIwFEBs9sc32i/rW5o6dSoaGxtxzz334KKLLkJhYSGuvfbaWMvWbcgt8gF4n59upgkAnx6qxjenGnzMHyVTVgxxwIim3JhSiphcvHMLX+mnlZjeXVJ+QYHpdTiOC3iYG61OEOJTjgYr39HnXx/P+hQEho6Wi8RxHL+NmFA/oNeo2ASgUCa8uIFIvBAAkQX8GPPTgJ8nsLZfF+aZ0Av3tdHqxDOfHcMdm77FTmEzj0Tx+WUtPU899RQA4Gc/+xmmTJmClpYWjB07NqaC9TRS9Bp4vAR2lxcGrQoPbNmHopwUPDz3AgCx8/kpaLENz/xWiRnPNmxs4pU/M1kniQ8MiSDST2E2+bapZvL47dQr3nTEt7e9NqBUl34Ou1APrw3DiJGC9g3QTTJoeW5I5hfFPgamGqBVc3AJzUb+w00pknTCxi2SCj+/PH8Qs1+Oz09rNlwegqVv7MEn913BZiT0Cp//gQceYPvoDR48uM8pPiAt8W3pcKHVzpu87ijl+YNBbB5TJaRFQGJ/cJBot9ZmmxNmo1ai/JYuMT+v/Aax8vs9zMHM5WCNONSCoemxaLpIOo2adfWJA34hmV/E9ukmHeu5yEzShUzX0o1bpMwvNfvFeX4f84dWXuq20fjDqIEpaLW7seNYXe/y+ZOTkzFv3jxMnDgR69evR0tLS6zl6nGIm3voDHyH2xu1VF8w6IIpv6BcBlGvOPX5j1a3wur0IM2k8zP7I2f+YO6Ff96aKr+YZf2DfYBv0aAKEy2fHwDbPdglzMmnzB9K+QeLzP5UoxapgrydNRH59/TbHG4YtCp2fyQ+Pwv4hX4m6N/R4qzxBWkA+FRtopj9sr6lxx57DKdOncLq1avx1VdfYfjw4bjxxhtjLVuPgjG/3c121nG6vZ029nQHYvPYyJQ/kPlTDVpcbEnHZ4f5oFGasfs+P50/6J9epDBoVSzyny1X+QXmDzb4pKugPr9LKB6iPn9WiOAdTXmmGjTCiK7wlgJFitDWS2F1eJAk3sYsmM8vw+ynyk9Ts3RwKiBd4OOBiMKNV111FcxmvvXwr3/9K958882YCBUPsGk+Dhebn+9we+AS8vzBduzpLqTML6399/cHfzqxgO0ARAd/aAQ/titR49smD8WkoZmSngDxw5ydYmCbjIhr6IMpv54xv1v4XNE0+1VwCMVDOo0KRTnJ0Ki4gJ2JKGjAj5rr8plfg7o23zDSeqsDZqM26E5G8lJ9/HvU5x8m7ELU0uHkW7i16qBtzj0JWU90bW0tnnnmGRQXF+Pmm29GSUlJr0j1RQJxZ5+E+T2xY346wRcIYvb7Kf+c4lymeGkmLTiOg2VAEsbmdW2sU5pJJ9kWCvAtRmoVx+QA+MwANXGD+/zCBhxCeiyazK/TqJhFoVWrMG1kNg4+fjWGCdNt/GHQqjE4w8gmFFGfP1Saj0Js9rc73Dhdb8UFg1KDjjSn+waGHeYhsDr1+S0DTOA4H/PH2+QHZDL/6NGjsXDhQrz00ku45JJLYi1T1CC3sQfwTd8932KX+vze2Pv8dHY+4GMsmvqjMGjVWDAhDxu/PMPYbPMvLwlbXx6xPHQTTZ1aMmDTqFMjWa9Bk80VkOMHwIaRUuaP5r3Sa1Ss/kKrVoHjuE4Lml6++WL2WWhaUg7zOz38dugHKltACDAu39xl5qepPrqgDEjWI9XAj2TrcHniHuwDZCp/RUWFrFx5okFuYw8AlObzAZndZxpZJRs/nIKW98Yu2k9n5wO+ST/B/MElVwxDbZsDU4v4PRQ6e6AjhW8HXY3ElTDp1Eg28Mof1OdXS33+aC9ItggDiXQ3IcBn9svx+QFeWfdVNgMASvLTfNWLngiLfERFWnQ8ON2cw+n2xD3NB3Si/Fu2bEFZWRnbpdcfd911V0yEigfMJi1G5qRg95lG1mzj9hJWnhkbs59/QEx+aT0VJ90QhCLXbMS6n02IuhwUepHyixtojFq1MIykg9UHiEE/B1XSaC6U4oUknJkdCjSgmdNJy7Ov0MeF/ZXNUHH8notBzf4I8vy8DHxtRJpRy+IK0V64u4Kwyn/w4EGUlZVh9+7dAe/1hh17IsXFQ9Ox6etyyWs2F32gYxDtp8wvUv68NCM+/tWUiFp0oy2PP/MbdWrGjP4dfYBPCZh5roluwM8nX+Tn/cnFg5Fq1GD84LSwx0mYv6IFhdnJzPXRqDhpnl+G2S9mdmrNmU06HK9th06jSnyz//HHHwfAT/QZMEAaHKqvr4+dVHHCxZaMQOUXHuhYlPdSJjNppV/DqIHBp7jGGpTlknRqyaBLk47fPAQIEe33C/hFMzMi3sCiK/UDGUk63Dip8zFilPnPNFhxrrkDCy/0uYo6odaAgpr94af3ipmfV/40oxY2pwcOtzfudf2AzGj/rFmzZL3W2zFxaAb7P40Ws8KVWPr8CcACgL/P72f2U+Y3BQb8dH6pvmi19IrPzZ83+t8BBa0H+PIET2rjRJaCTqOCUzSJ19HJRGdA6vPT3ZzEU5MT4TsPu/y43W44nU54vV50dHSwppeWlhbYbLYeEbAnkWs2Ij/diMqmDgzPSkZNq4PVYfeUzx9PUHmS9RpWTAPwixM192mjixhU2ZnPH4MMhP//ow1q9n+wj9/8U+wm+Pc8uNz8lu3hnglxwFbM/BSJEPALezdXrVqF5ORk7N+/H0lJSUhOTkZycjIuuOCCPlfhRzG5cAB0ahWKcviIsZU90DH0+RPgQQCki5HU7FfjtsnDsHbRuKD5co1aBRUXm2i/2LSO5nn9Qd0Zp8eL5TOLJPUT/ma/0+PtVBa6Xx/gGzoinn2YCAt+WOZfuXIlVq5ciTvvvBN//vOfe0qmuOJ/51yAWy6zsP5r6vPHsqsvEUxAwKdcyX4BP4NWjYJMEwrClBHrNCo2NDOaC2V3A35yUZyXhnuvKsSMC3IkJj+Vwb+fX44sei2/aNCAX7D9EuIJWUvprbfeira2NvZ7W1sb9uzZEzOh4gmzUYsLclNZoMkag/QVBS2DTQQWAIKn+ui47M4gZsKoNvaIzxtDs1+nUeH+WSMDFB8INPv5vQk6/86oRUdLtiXj07S9JOC3dOlSmEy+Vd9oNGLp0qUxEyocvvzyS9xxxx244447MGjQIBw/fjwm1/HlrnvC54//gwBIzX4qk9yFSSeJyve+gF840OaiHcfqsO7zE7zyy/iMNOJPN3ENNjsxnpD11Hm9XqhFvcsajSZgF59wWLZsGf7xj3/g7NmzOHDggGQewPHjx3HLLbegvr4eaWlpeO211zB69OiQ57r88stx+eWXo7q6GufPn8eIESNkyxEJWLMKTV/F0udPgAcBkJr9lPnlxiP0MVJScaovlj5/OFCf/8Udp/DfE/VIN2mDpjz9QSP+NEgq3Swl/t+5rLup0+lw8uRJ9vuJEyfYXD85WLhwIf773/9iyJDAfOvSpUuxZMkSHDt2DCtWrMBtt90GAKiursaVV14p+RFPDF63bh3uvPNO2TJECso4tP0ypsyfIAE/rSTVx/OC3IdUzNC9MdrfmQwOt5c16TTZXLIWuHDMnwhBXlnMv3LlSkyePBlz584FAHzyySd45ZVXZF/kiiuuCPp6bW0t9u7di08//RQAcP311+Oee+7BmTNnYLFYsG3btqB/Z7fbsWPHDvz2t78N+v7atWuxdu1a9ntX9hX0+fw04NcP8vyswk/NzFK5LonYMoruJJ/4m/06tVT5AXkLEU33+Zg/scx+WXdz7ty52L59OyZMmIAJEyZgx44dmD17drcvXlFRgUGDBkGjobPYORQUFKC8vDzs323atAk//elPQ5YYL1++HJWVlewnOTl4+2c4BNSrx8Dsp5NoO9veu6dQWpCGwRlGXJCb6mN+mQwlUdIoMrS+h6L94UCj/eKgnxzlz00zIC/NyIhEq1ax1vFEWPBlR5qKiopQVNT5VtSRwl+BaSFRONx+++2yzh1JS68/fD5/7FJ9lw7PxG+vHYN543Kjfu6uYEJBOr5YcRWAwD3kO4PYH4/mvZKY/XHz+X33wKDlU5pyrJDfXjtWsukJwLN/u8OdEEFeWXczVtt1DR48GJWVlSx4SAhBRUUFCgoKun1uILLR3f5gs+jdsevqU6s43HypJSEeBH8YtWqYjVoMDNJdGAyxMs91PZTqkyvDnGJ+oZbTYWg2agN2T04LMjU5XpB1N2O1XVd2djbGjx+PTZs2AQDeffddWCwWWCyWbp8biGzTDn/4f7nRDGL1BnAch7/dfTn+d84oWcfrutmAE/q8icD8vuv+eHwedBpVwN4FcpEWYlJTPCDrbnZ3u667774b+fn5qKysxIwZM1BYWMje27BhAzZs2ICioiKsXr06okBiZ4gG81PEwuxPdAwdkBS0kScYpEU+0ZzkE5tFJTIZfNcdmZOCt5Zcgv/90QVdOhedhJQIzC9r+fLfrisnJyeiGX7r1q3DunXrgr43cuRI7Ny5U/a5egp6vwquWJj9fQmxyvNLU33xC/gB/AI3IFmPbJmuUDAMzjAxlyrekKX8/tt16fV6lJWVxVq2biMaAT+KeLFOb4Ek1Rejxp54pvoAYFCaodsTd++9qhA3XDxYMiMxXujT23VFMsPPH/7KrxB/eEiLfKI7wJMibsovyJCX3v05lnzfRPwVH4hwbj/AR+gHDx4cC1kSCmKzn98tVtH+cIhZtD9BKvwA356JfQWJsQTFCN0x+2PFZH0V4n3r+lpjDzX789Mj3xkpkdGnHdloRftjUdrb19ATef54p/r6GvMrT3UIqFUcS+8pkf7OIV0so+jza2NjUUSC0sFpsGSaJDMe+wLCmv0rVqwI+8c0EJio6I7ZD/DBJrfTE7eHrjeB9rdr1dGNj4h3NYrXIjw2z4xtD06Ly7VjibDKn5TU87Pjo4nuRPsBYZ84p0dhfhmgzB9tF4kuvDphqy4F0UOnM/z6M/iIv0vx+WWAMnS0g6Mcx0GvUcXN3+/LkB3tf++99/D999/Dbrez1xLd7O8uGJspZn+noLX9sVBSnUalFFnFALLu6H333YeNGzfi5ZdfhsfjwVtvvYWGhoZYy9ZtdKexB/AVmChmf+eI5UKp16iUuEsMIEv5t27dir///e/IysrCM888g927d6O2tjbWsnUb3Un1AWI/VnnwOgO9V7FgaJ1aYf5YQJbZbzAYoFLxAReXy4WcnBycO3cu1rLFHfoYBbH6Iqi5Hwsl1WvVUGJ90Ycs5U9JSYHNZsPkyZNxyy23YODAgREN8OytUHx++aAdd7Ewz68eMzDq51Qg0+zfvHkzNBoN1qxZgzFjxkClUmHLli2xli3uoPX9is/fOWh5byyspF//aBR+/SN5Q0UUyIcs5s/JyWH/f/jhh2MmTKKBmv2x2KG3r4H5/HFqvlEQOWQp/5EjR7Bq1SqcOnVKslnHrl27YiZYNNDdCj+dEu2XDab8yr3qNZCl/IsWLcLNN9+MW2+9VbJzT6KjuxV+1OxXfP7OEcuAn4LYQPYYrwcffDDWsiQclFSffCjB0d4HWcv07Nmz8c9//jPWsiQcfEU+Cpt1BnqvlDLc3gNZzD99+nRce+21UKvV0Ov1IISA47heUejTHegV5pcNbYxq+xXEDrKUf+nSpXjttdcwYcKEXuXzdxd6xZSVDZ/ZrzB/b4Es5c/MzMTChQtjLUvCQfH55UMx+3sfZH1TP/7xj7F+/Xo0NjbCZrOxn0RH9xt7aJGP8kB3hiS9BvdeVYhFF/X94a59BRyRsTOmSvTwcxzHfH6PxxPmrxIHdLegSPF/X53Byn8cwg0XD8bq60tiIJkCBdFDpM+5LLPf6/V2flAfhNLSq6AvQ7Fnw0Dx+RX0ZYRl/unTp2Pr1q3IysqSzE/rP6k+WuGnrJEK+h7CKj/dOnvPnj09IkyiQcnzK+jLCKv8ubm5AIAhQ4b0iDCJBqWxR0FfhqyA3969e/HQQw8FdPWdOnUqZoIlAvRK4YqCPgxZyn/LLbfgnnvuwaWXXtqvKvyUgJ+CvgxZyq9Wq7F06dJYy5JwyE83waBVYVhW7968RIGCYJCl/Jdffjm+++47jB8/PtbyhEVVVRXuvPNO5OTkoLm5GRs3bozprkJZKXocenw2FOJX0Bchy5n96quvcMkll6CkpAQTJ05kP3KwbNkyWCwWcByHgwcPSt47fvw4LrvsMhQVFWHixIk4fPhw2HMdOnQIU6dOxYsvvoi8vDycPHlSlgzdgVoV3b3nFChIFMhi/meffbbLF1i4cCFWrFiByZMnB7y3dOlSLFmyBIsXL8Y777yD2267DTt37kR1dTVuuOEGybGlpaV47LHH8Mwzz2Dbtm0wGAwYM2ZMl+VSoKC/Q1ZtfzRgsVjw4YcfYuzYsQCA2tpaFBUVob6+HhqNBoQQ5Obm4uuvv4bFYgl6jrVr12LYsGG47rrrsHbtWgwfPhzXXntt0OPWrl3Lfm9vb0dzc3NMPpcCBYmCqNb2/8///A/+8Ic/oKysLKjp+/bbb0cuoYCKigoMGjQIGg0vAsdxKCgoQHl5eUjlnzNnDh566CF89tlnOHfuHH72s58FPW758uVYvnw5+72rM/wUKOjLCKv81FSfN29eTC7uv6B0ZoSMGjUK7733XsTXqaur63QBaG9vR3JycsTnjgd6i6y9RU6gb8haV1cX0XnCKv8111wDgM/zRxuDBw9GZWUl3G43M/srKipQUFAQ9Ws5HI5Oj+lq22880Ftk7S1yAv1T1rDKv2LFirB/3J0turOzszF+/Hhs2rQJixcvxrvvvguLxRLS5FegQEF0ETbV9/TTT2P79u0wGo1ISkoK+JGDu+++m61UM2bMQGFhIXtvw4YN2LBhA4qKirB69Wq88sor3fs0ChQokA8SBlu3biU///nPSWFhIXnooYfIiRMnwh3eq/HMM8/EWwTZ6C2y9hY5CemfsspK9bW2tmLz5s3YuHEjjEYjVq9ejUmTJvXE2qRAgYIYQVaFX2pqKubPn49rr70WR48exdGjR2MtlwIFCmKMsMrv8Xjw/vvvY968eZgxYwY4jsPevXtjEv1XoEBBzyKs8ufl5eH3v/895s2bhy1btmD+/PloamrC4cOHO63D7y2ItL+gp2C323HdddehqKgIpaWlmD17Ns6cOQMAuPLKKzFs2DCUlpaitLQUf/zjH+MrLPgKzlGjRjGZ/vrXvwLgKzlnz56NESNGYOzYsfjvf/8bVzmbm5uZjKWlpSgqKoJGo0FjY2Pc72uoPphw99Bms+GnP/0pCgsLUVRUFFkdTLiAwJAhQ4jFYiEWi4UMHTqU/Z/+3hcwbdo0snHjRkIIIVu2bCGXXHJJfAUS0NHRQT766CPi9XoJIYQ8//zzZObMmYQQQqZOnUo++OCDeIoXgCFDhpADBw4EvP6LX/yCrFy5khBCyK5du0hBQQFxuVw9LF1orFmzhsybN48QEv/7un37dlJRURFwL8Pdw8cff5zccssthBBCTp06RXJyckhjY6Os64VV/r6OmpoaYjab2Y30er0kJyeHnD59Or6CBcHu3bvJ8OHDCSHxf0iDIZTyJyUlkdraWvb7xRdfTD7//PMelCw8Ro8eTd5//31CSOLcV/97Ge4ejh49muzatYu9V1ZWxsisM/Tr+VTh+gsSDc899xyruASABx98EMXFxfjJT36SMOPUbrzxRhQXF+P2229HXV0dGhoa4PV6kZWVxY6xWCwJc3937tyJhoYGSfl6ot3Xzu5heXm5ZMZmJPe3Xys/EHl/QTzw5JNP4vjx41i1ahUA4I033sCRI0ewf/9+TJkyJWa9F5Fgx44d2LdvH/bu3YvMzEwWFE7k+/vqq6/i5ptvZot/It5XoPN76D9WXza6baP0YtTU1JDU1NSENvvXrFlDLrzwQtLU1BTyGL1eT+rr63tQqvCoqqoiycnJhBBCTCZTQpr97e3tJCUlhRw5ciTkMfG6r/5mf7h7qJj9XYS4vwBAwvUXrF27Fps3b8Znn32GtLQ0AIDb7UZNTQ075t1330VOTg4yMzPjJSasVqtkXsLmzZvZyLeysjKsW7cOALB7925UV1cHHezS09iyZQtKSkowatQoAIl5XynC3UPxe6dPn8b27dsxf/58eSeOylLVi3H06FFyySWXkBEjRpALL7yQHDx4MN4iEUIIqaioIADIsGHDyLhx48i4cePIxIkTSXt7O7nwwgvJ2LFjSUlJCbnqqqvI999/H1dZT548SUpLS0lxcTEZO3YsmT9/PrOeqqurycyZM0lhYSEZPXo02bZtW1xlpZg8eTJ59dVX2e+JcF/vuusukpeXR9RqNcnJyWEB3nD3sL29nSxatIgMHz6cjBgxgmzZskX29Xpsko8CBQoSC/3a7FegoD9DUX4FCvopFOVXoKCfQlF+BQr6KRTlV6Cgn0JRfgUK+ikU5U9wWCwW1t752muv4dixY1G/RnNzc8Aw1ttvvx1ffPFF1K/VEwj2eRQEQlH+XoSuKr/X64XX6w35fjBlefnllzFlypSIr5UI6I7yu93uKEuTuFCUv5fg5Zdfxp49e7Bs2TKUlpbi448/BsBPWJ44cSImTJiAOXPmoKKiAgDw2GOP4aabbsKCBQtQWlqK8+fP48EHH8TFF1+M0tJSTJ06FcePHwcA3HHHHWzIxUUXXQSAHxjy4YcfAgBqamrw4x//GMXFxRg7dixefPFFJpfFYsHjjz+Oyy67DEOHDsUTTzwR8jNs3LgRpaWlGDduHC666CI2nOSNN95AcXExSkpKMHfuXJw7dw4Av9gtXLiQ/f2HH36IK6+8EgCwbds2lJaW4q677sK4ceMwZswY7NmzJ+Tnqa6uxqJFizBx4kSUlJTg0UcflXyGVatWYdq0af1rSlWUKxQVRBniJg//fvM333yT/PKXvyRut5sQQsjrr79O5s+fTwghZOXKlSQvL4/U1NSw4+vq6tj/N2/eTObOnUsIIeT06dMkMzNTcl3xtRYtWkR+/etfE0L4Zqj8/HzyzTffMPnuu+8+QgghtbW1JDU1lVRWVgZ8js8//5wMHz6cVFVVEUIIsVqtxGq1kgMHDpCcnBz2N0888QSZM2cOIYSQjRs3kuuvv56d44MPPiBTp05l59NoNGT37t2EEEL+/Oc/k1mzZoX8PLNmzSLbt28nhBDicrnIhQ2yfQAAAr9JREFU1VdfTd577z32GZYsWcIGp/QXyNqlV0Fi4m9/+xv27NmDCy+8EAA/c1GtVrP3582bh+zsbPb7p59+iueffx5tbW3wer1obW2VdZ1///vf2LdvHwC+GWrBggXYunUr26b9xhtvBABkZWVh2LBhOH36NPLy8iTn+Oijj3DzzTcjNzcXAGAymQAAn3/+OebNm8eOv+uuu/DEE0/Iak0dOXIkY/ZLL70UTz/9dNDjrFYr/vOf/0gad9rb2yWDaH/xi1/0u63YFeXvxSCE4De/+Q1uvfXWoO+L93MrLy/HsmXLsGvXLgwbNgz79+/HVVddJfta/ooh/t1gMLD/q9XqiPxmQojkXOL/azQaeDwe9rvdbpf8rdzrer1ecByH3bt3Q6vVBj2mt+zTF00oPn8vQmpqKlpaWtjv8+fPx5/+9Cc0NjYCAFwuF7777rugf9vS0gKdToeBAweCEIIXXnhBcl6bzRZSeWbMmMH8/Lq6Orz//vsRLRwAv+/j66+/jurqagD84EmbzYbp06fj448/Zq+vX78e06dPB8dxGD58OPbt2we73Q63242//OUvsq7l/3lSUlIwZcoUrF69mh1TVVXVa/bmixUU5u9FWLJkCe6//36sWbMGTz75JG666SY0NDTgyiuvBMdxcLvduO2221gvvRjFxcUoKyvDmDFjUFBQgJkzZ7L3MjIy2AiupKQkFjijeO6553DHHXegpKQEXq8XDz/8MDP55eKKK67Ab37zG8yaNQscx0Gn0+Gdd97BmDFj8Pvf/x6zZs0CwG/gSheaSy+9FFdffTXGjh0Li8WC8ePHs4UuHIJ9njfffBPLly9HcXExAJ7p169f36+3b1daehUo6KdQzH4FCvopFOVXoKCfQlF+BQr6KRTlV6Cgn0JRfgUK+ikU5VegoJ9CUX4FCvopFOVXoKCf4v8DwUnGMYmKgkUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 240x160 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure(num=None, figsize=(3, 2), dpi=80, facecolor='w', edgecolor='k')\n",
    "plt.plot(active1[0:100])\n",
    "plt.yscale(\"log\")\n",
    "plt.xlabel(\"Iteration counter\")\n",
    "plt.ylabel(\"Minimal activation \" + bitsString)\n",
    "plt.legend()\n",
    "plt.savefig('minActivation'+bitsString+'.pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Investigating the shape of the bifurcation zone\n",
    "### Initialize networks in the exact same way and train in 32 bits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Initialize random seed\n",
    "np.random.seed(12345)\n",
    "torch.manual_seed(123)\n",
    "\n",
    "\n",
    "torch.set_default_tensor_type(torch.FloatTensor)\n",
    "torch.set_default_dtype(torch.float32)\n",
    "\n",
    "## Load Training data\n",
    "\n",
    "batch_size = 128\n",
    "\n",
    "transform = transforms.ToTensor()\n",
    "\n",
    "train_data = datasets.MNIST(root='data', train=True,\n",
    "                                   download=True, transform=transform)\n",
    "test_data = datasets.MNIST(root='data', train=False,\n",
    "                                  download=True, transform=transform)\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size)\n",
    "test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==> Building models..\n",
      "==> Copying models initial parameters..\n",
      "Sequential(\n",
      "  (0): Linear(in_features=784, out_features=2000, bias=True)\n",
      "  (1): MyReLU()\n",
      "  (2): Linear(in_features=2000, out_features=128, bias=True)\n",
      "  (3): MyReLU()\n",
      "  (4): Linear(in_features=128, out_features=10, bias=True)\n",
      ")\n",
      "Sequential(\n",
      "  (0): Linear(in_features=784, out_features=2000, bias=True)\n",
      "  (1): ReLU()\n",
      "  (2): Linear(in_features=2000, out_features=128, bias=True)\n",
      "  (3): ReLU()\n",
      "  (4): Linear(in_features=128, out_features=10, bias=True)\n",
      ")\n",
      "Sequential(\n",
      "  (0): Linear(in_features=784, out_features=2000, bias=True)\n",
      "  (1): ReLU()\n",
      "  (2): Linear(in_features=2000, out_features=128, bias=True)\n",
      "  (3): ReLU()\n",
      "  (4): Linear(in_features=128, out_features=10, bias=True)\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print('==> Building models..')\n",
    "net1 = torch.nn.Sequential(\n",
    "    torch.nn.Linear(28*28, 2000),\n",
    "    MyReLU(),\n",
    "    torch.nn.Linear(2000, 128),\n",
    "    MyReLU(),\n",
    "    torch.nn.Linear(128, 10))\n",
    "\n",
    "net2 = torch.nn.Sequential(\n",
    "    torch.nn.Linear(28*28,2000),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(2000, 128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128, 10))\n",
    "\n",
    "net3 = torch.nn.Sequential(\n",
    "    torch.nn.Linear(28*28,2000),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(2000, 128),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(128, 10))\n",
    "\n",
    "\n",
    "\n",
    "print('==> Copying models initial parameters..')\n",
    "params1 = net1.named_parameters()\n",
    "params2 = net2.named_parameters()\n",
    "params3 = net3.named_parameters()\n",
    "\n",
    "\n",
    "dict_params2 = dict(params2)\n",
    "dict_params3 = dict(params3)\n",
    "\n",
    "for name, param in params1:\n",
    "    if name in dict_params2:\n",
    "        dict_params2[name].data.copy_(param.data)\n",
    "    if name in dict_params3:\n",
    "        dict_params3[name].data.copy_(param.data)\n",
    "\n",
    "nets = [net1, net2, net3]\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "for net in nets:\n",
    "    net.to(device)\n",
    "    print(net)\n",
    "\n",
    "optimizers = []\n",
    "lr = 0.01 * (npr.random(1)[0]/5.0+1) \n",
    "for net in nets:\n",
    "    optimizers.append(torch.optim.SGD(net.parameters(), lr=lr))\n",
    "    \n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Retrain and stop at bifurcation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                \r"
     ]
    }
   ],
   "source": [
    "for net in nets:\n",
    "    net.train()\n",
    "\n",
    "t = tqdm(iter(train_loader), leave=False, total=len(train_loader))\n",
    "NstepMax = t.total \n",
    "\n",
    "## Difference between weights\n",
    "difference12 = np.zeros(NstepMax)\n",
    "difference23 = np.zeros(NstepMax)\n",
    "\n",
    "## Activation of the first layer before application of relu\n",
    "active1 = np.zeros(NstepMax)\n",
    "active2 = np.zeros(NstepMax)\n",
    "active3 = np.zeros(NstepMax)\n",
    "\n",
    "for batch_idx, (inputs, targets) in enumerate(t):\n",
    "    ## Forward path for each network\n",
    "    for i in range(len(nets)):\n",
    "        optimizers[i].zero_grad()\n",
    "        inView = inputs.view(-1, 28 * 28)\n",
    "        outputs = nets[i](inView)\n",
    "        loss = criterion(outputs, targets)\n",
    "        loss.backward()\n",
    "        \n",
    "        ## Keep track of activations\n",
    "        active1[batch_idx] =  torch.min(torch.abs(net1[0](inView))).detach().numpy()\n",
    "        active2[batch_idx] =  torch.min(torch.abs(net2[0](inView))).detach().numpy()\n",
    "        active3[batch_idx] =  torch.min(torch.abs(net3[0](inView))).detach().numpy()\n",
    "    \n",
    "    ## Keep track of weight differences\n",
    "    sum1 = 0\n",
    "    sum2 = 0\n",
    "    for j in [0, 2, 4]:\n",
    "        sum1 += torch.abs(net1[j].weight - net2[j].weight).sum()\n",
    "        sum1 += torch.abs(net1[j].bias - net2[j].bias).sum()\n",
    "        sum2 += torch.abs(net3[j].weight - net2[j].weight).sum()\n",
    "        sum2 += torch.abs(net3[j].bias - net2[j].bias).sum()\n",
    "                \n",
    "    difference12[batch_idx] = sum1\n",
    "    difference23[batch_idx] = sum2\n",
    "    \n",
    "    if active1[batch_idx]==0:\n",
    "        break\n",
    "    \n",
    "    ## Perform an otptimization step for each network\n",
    "    for i in range(len(nets)):\n",
    "        optimizers[i].step()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "117\n",
      "267\n"
     ]
    }
   ],
   "source": [
    "## Print the problematic index \n",
    "tempWhere = np.where((net1[0](inView)).detach().numpy() == 0)\n",
    "id1 = tempWhere[0][0]\n",
    "id2 = tempWhere[1][0]\n",
    "print(id1)\n",
    "print(id2)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Look in a randomly generated hyperplane"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import copy\n",
    "torch.manual_seed(123)\n",
    "\n",
    "perturb1 = torch.rand(net1[0].weight.shape)\n",
    "perturb1 = perturb1 / torch.norm(perturb1)\n",
    "perturb2 = torch.rand(net1[0].weight.shape)\n",
    "perturb2 = perturb2 / torch.norm(perturb2)\n",
    "net1bis = copy.deepcopy(net1)\n",
    "weightSave = net1[0].weight.clone()\n",
    "\n",
    "if doublePrecision:\n",
    "    torch.set_default_tensor_type(torch.DoubleTensor)\n",
    "    torch.set_default_dtype(torch.float64)\n",
    "    inView = inView.to(torch.float64)\n",
    "    perturb1 = perturb1.to(torch.float64)\n",
    "    perturb2 = perturb2.to(torch.float64)\n",
    "    weightSave = weightSave.to(torch.float64)\n",
    "    net1 = net1.to(torch.float64)\n",
    "    net1bis = net1bis.to(torch.float64)\n",
    "    net2 = net2.to(torch.float64)\n",
    "    net3 = net3.to(torch.float64)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "N = 100\n",
    "\n",
    "x = np.arange(-N,N) / N * 5 / 10**6 \n",
    "y = np.arange(-N,N) / N * 5 / 10**6 \n",
    "X, Y = np.meshgrid(x, y)\n",
    "Z = np.zeros((x.size, y.size))\n",
    "for i in range(x.size):\n",
    "    for j in range(y.size):\n",
    "        net1bis[0].weight = torch.nn.Parameter(weightSave +  x[i] * perturb1 + y[j] * perturb2)\n",
    "        Z[i,j] = torch.abs(net1bis[0](inView)[id1,id2]).detach().numpy()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAO8AAACzCAYAAABhPEs1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAMTQAADE0B0s6tTgAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAaQElEQVR4nO2df1BU1xXHvysggihqN6Dyo1h+RYssUTFUo+hgJ0K02iiMWjPRkEEqTqbBsUkz01b6w6Q17kx1pJiMY2t0bJs2himJmfgjgEapMIQ6QGIWf28qGpQowoLA3v5Bdt1ld9m3u+/tu2/f+cy8GX17332X3f3uOe/cc+/RMMYYCIJQHKPkHgBBEN5B4iUIhULiJQiFQuIlCIVC4iUIhULiJQiFQuIlCIXChXhfeuklJCQkQKPRoLm5WdS+y8vLMX36dKSlpSE9PR29vb2i9k8QcsGFeFevXo0zZ87gu9/9rqj9VlZW4vDhw6irq0NzczNOnDiBkJAQUe9BEHLBhXgXLlyI2NhYh/MGgwHPPPMMMjMzodPpUF5e7lG/O3fuRFlZGSIjIwEAUVFRCAoKEmXMBCE3XIjXGYODg1i3bh127dqF+vp6nDt3DhUVFWhsbBTcR2trKxoaGjB//nzMmTMHu3fvlnDEBOFfguUegCsuXryIlpYWrFmzxnquq6sLra2tmDVrFhYsWIDPP//c6bWfffYZ4uLiMDAwgEuXLqG2thb37t1DdnY2kpKSkJeX568/gyAkg1vxMsag1WrR1NTk9PXTp0+77SM+Ph5r165FUFAQJk2ahNzcXJw/f57ESwQE3LrNqampCA8Px8GDB63n2tracPfuXcF9rFu3Dh999BEAoLe3FzU1NdDpdKKPlSDkQMPDksCSkhJUVlaivb0dWq0WERERaGtrg8FgwMsvv4zr169jcHAQjz32GA4fPoyYmBhB/ZpMJmzatAkNDQ3QaDTIz8/H9u3bpf1jCMJPcCFegiA8h1u3mSCIkSHxEoRCkT3aHBwyGuMiJ7l8fZTG+741Gh8uFoAvY5MSDTgdmAQ4+4g7O++gr69vxOsyZ8/BzfZ2h/NTpk5BfX29WMOTFNnFOy5yEl4/VO223cRQ79MaI0Oly6oaHyr7W+iUcZyOSwrCh/2t2Ysz3F5zs70d17684nA+Kn4yCgoKkJ+fj/z8fNHGKAWyf8JmMHT2DWCimy9bZ1+/1wK+1zcomYDv9w0A4E/EXX0DqhFwz7efwXARe0NYWBj+8Y9/+NyPP+Dmmbfz2w9g5Db9Xvd/r2/Q62uFcF/A+P1NV98Aujgcl1T0iPC3mkwmFBQU4N133xVhRNLC1U+zUAsMeOdGWwQspRXmzQID6rTC3kKW1wc6+wYUbYXv9w1wa4UJ95Dl9QCzixQRf1hhKQNZPFphi4DVYoVHgoGhr9/xB40sr4e4srZCLPBQO++s8L2+QbLChGKRRLxlZWVebWnjSsBKdqMBCmYR0iC6eBsbG1FXV4f4+HhB7c0MuGV6JDxXQhUqYJ6tMI+QgO1R0jOvqOLt6+tDSUkJysvLBWc3DX67LmK4gAPRCpMbzT+WZ17eEzQAkcX7q1/9CuvXr8e0adM8uu626SEAewEDvlthbyE3WgUw4OHDQYdDSYgm3nPnzqG+vh6bN28esZ1er0dsbKz16Df1oMPUbydgtbjRvIqYUAaiibempgZffPEFpk2bhoSEBBiNRjz99NM4duyYXbvS0lIYjUbrETQmDJ3d/VYBO7PCgepGA/xaYbWipGdeyRbjJyQkoKqqCmlpaSO2GzMxCotfP4oOUz86u/sxcWwItGEhiAobbW0THfZoHtfV3K+7OeFH7WiBgycocU44d+kcGI3GEdvETI1BS9NFh/Npsx53ey0vyP7JBGk0dkLt7La3klFho3HL1G8VsMXSDherq/PDoQUOnqGm1EqlIdmncvXqVY/aOxOwNiwEt00PHQQMuM7AUvIKJYAys/wFY0CvwgJUw5H90xilGXKLb5n6rQLWhoWgw9QPQ0cPJo4dEplFwADsrLAvAgZogYMnkBWWjt/+9rf4+uuvERISgj/84Q8IDnb/PnORHgk8EmRU2GhEhY2GNiwEE8eGqDaYRdFovhmpOJ7BYMC8efOQkpKCuXPnorW1dcS+qqqq0NLSgrFjxyI6OlqQcAEOxGu7lUx0WIidiLVhIUjWhgMADB2OU0q2BPKUEm+obk7YCSMVx9u0aROKiorw5Zdf4uc//zkKCwsBAO3t7Vi0aJHd8bOf/QwtLS1ITU3F66+/jq6uLtTU1AgaAxc+0MTQYDuRDXejLdgGs1y50Zb+7K6jYJYkqNmNXrhwodPzt2/fRmNjIz7++GMAwKpVq7BlyxZcvXoVCQkJqK6udrjmb3/7G27dugUA+M53voP79+8LGoPsltfCxNBgO3E5s8JC3GiAMrP8SaBZ4QcPHtglEen1eo+uv3HjBqZOnWp1fTUaDeLj43H9+nWX1zz77LO4cOECtm7digsXLmDp0qWC7sXdz6Y7K0zBLApmiYGZMXSbHH+og4KCMG/ePJ82oBue1+8ulWL06NHYv3+/x/fh8t22iMsiYiFuNEBzwnKjNAE7w9fF+HFxcTAajRgYGEBwcDAYY7hx44bgVXaeILvbPAoaQVlTSg1mSQm50eLja3pkVFQUnnjiCRw6dAgA8K9//QsJCQlISEgQcZRDyC5eC0LTHm0FDMDpszDgfIED7dbhP5QsYCGUlJQgNjYWRqMRS5YsQVJSkvW1ffv2Yd++fUhJScEbb7zhlUssBNkLjU3UTnbYdF2IpbQV5m3TQ4fcaOCRwG0zswDf8qN5zY0G+HOjLfjblRaS2zx58lR8evqCw/nsxRmKyW3mxvLaIoUbrZZlhjzCoxVmbGib2OGHklYV8flTDceglavzFMyyh4JZvkG7R4qIL1a4s3toSgmg3Tp4QAnBLLK8HiCk0t7wuV9X522t8G3TQyRrw1U/JwyQFfYEsrweIuSLPzwDy9V5WuDgCK9WmPAN0cTb29uLlStXIiUlBRkZGVi6dKlHa3onhoYIFrG780qdE1ZbMEtON5qBWe9veyjJbRbV8hYVFeHixYtoamrCsmXLUFRU5HEfYgkYEGaFAc/mhJVshWlO2D2q3Pp1zJgxyMvLs+Z1ZmVl4fLly171JbYbTQsc7OFVwDyJWAlI9sy7e/duLF++3OvrxXSjAdCc8DB4FDAgvxVWktssSchvx44dMBgMqKiocHhNr9fbLbPqNfWM2NfE0BC3AqA5Ye+gaLQjqo42v/nmm3jvvfdw7NgxhIeHO7w+fN/mMWGObYZDwSz1WWGp3WgzexQDsD2UhKji1ev1OHLkCI4fP44JEyaI2TUACmZRMIuwRTTxGo1GbN26Fd988w0WL16MjIwMPPnkk2J1b4WCWRTMkhJVPvPGxsa63TFALCwCFvIs7GlmFkC7dah5tw5VP/N6itBSoM6gzCxyo9WM7OIFhiyIt1bEn8EsQLoFDrRbh3BoTngILsRrwRc3UKpgFvBotw4hc8K0W4f/8EXAjDHr+2Z7KAmuxAv4R8BSutGAc7Eq3Y0GyApLxaeffori4mIUFxdj6tSpMBgMgq7jTryActxoysziB38LWMxyJ/Pnz0dFRQW2b9+OzMxMJCcnCxoDl+K1IKcVtoWCWfYEohvtKWKWO7Gwd+9e/PSnPxU8Bv7mA4ZhEbA3X0YhqZVD7RynlJylVgJwmFKyBLKoHCkf+CpgS8UEC6WlpSgtLXVoJ2a5E2BoSW1tbS1+85vfCB4rX+/8CESGBnktYECaOWHarYNPAQvBzJx/JyIiInzaPXKkcicj7d186NAhrF271qOpU0W9694KGKAFDmpc4OANlgwrf5Y7AYAXX3zR4/tw/czrDKUGs2yhYFbgYlvuBEBglzvxFh7nhGmBA7/BLH/hz3InivZzeHKjATi40u7caEsfFMziB6G5zSUlJaisrER7ezuWLFmCiIgItLW1ARgqd7Jhwwbs2LED48ePx1//+ldJxip7uRNt1BQcev+cz/34Yk2EWDGhrq7Fulos7vAyLLbithWxLyVYhtrxWYZFDgEXrMhyG3SK1Ebj1f0nHc6/+UI2cnJyfHrm9RfK+2l0AU9WmIJZj1BaMItWFckEBbMomOUrSlrPK6p4PU0LkwoKZlEwy1tUufUr4DotTA54WuBAu3XYw7uAlYJo4rWkha1fvx7AUFrYlStX3FZNEFKryFt4caMB2np2OHILeCjDasDhUJLbLFoUwdu0MGAomCHlh0nBLApmCUW1ASshaWF6vR6xsbHWo6enG8DQByjlh8iLFaZypI7IbYWVimjiFZoWNnzf5vDwsXavS/0rzGMwCwCSteGCd+twRiC40TyIWElus2jiFTMtzB9W2Ft42K0jUKPRgPxWWLXR5n379mHfvn1ISUnBG2+8gf379/vUn5rdaICCWcTIiKqO1NRUnDvne6qjLYEczBpp72gLFMyS5gfczBxjCkpDERlWvAezhLXzfU4YoGAW8QhFiNcCr8EsmhMOHDdaSQErfibYBGIRsFQfKM0Jq9ONtqDaeV5/wrMbTQscAsMK847iLK8tgRDM8qYQmgVbK+xq0ztLf3bXkRXGIGPWHz6loljLa0FNc8K0wMGeQLHC//vf/7BixQoUFRWhoKAA3d3dgq5TvHgtqMGNBiiYNRy5BCxmxYSWlhZkZ2fjrbfeQkxMDC5duiRoDAEjXkBdVhigzCwLcqRWilkxITMzEx9//DF+9KMf4auvvsL3v/99QWNQ9DOvK6R8Fva1ggMgzgbwrio4uNoAXsxN72z/Fk8IpE3vxKyYoNfrUVxcjJUrV0Kv16OqqgorVqxwO4aAsry28DonDPinHKkvu3UIQelutJkxdJj6HQ5LuRPLodfrPbr/SEtjXZGXl4eDBw+ipKQEtbW1ePLJJwXdKyAtrwWaE/Zu61mlR6N9ISgoCPPmzfNrxYTHH38c7733nsf3CVjLa4saglm0wIEPqGKCBASCG03BLOnxdUmgPysmqEa8AP8LHGi3DvkRmttcUlKC2NhYGI1GLFmyBElJSdbXxF4a64qAfuZ1hRozs6gcqT1m5hgT8IS9e/di7969Tl+TYmmsM1QpXoCCWWpd4OAO1S1MeO211zB9+nTodDrMnTsXp06dEqNbv6BmNxpQ7wKHQEAU8S5YsACNjY3473//i7fffhurVq1Cb2+vGF37hUAIZgk5LzSYpZYKDs5Q3Xre3Nxc679nzpyJwcFBdHR0IDY2Vozu/YI/3GjA+8wsf5Uj1YaF4LbpoarKkdqiOrfZlgMHDiAxMdGlcF3t28wLvFphWuDgHzdaSZZXUH3eBQsW4PPPP3f62meffYa4uDgAwMmTJ7Fx40YcP34cqampggYQHT0VRz8878GQ/YeUEWmp6wkPtXMvNFth3jY9dKgnDDxKvbS1woBvNYWlrCf8YsFTbuvzjhqvxcQSx6LXYe9scnstLwgyM6dPn3bbpqamBhs3bsS///1vwcK1MC40GF0crs2kBQ78LnAA/DOlxDOiuM21tbV47rnnUFlZCZ1O51Uf40KDMY6jmjUWeHWjASpHKoUbrSS3WZRvZmFhIfr6+rBx40bruXfeeQczZ870uC8erbCa5oQB18EsC4EczFJSwEoU8RoMBjG6sWKxwDyKONDdaMAxsWMkN9rSfuj+5Eb7E/78VBvICnsGZWYJt8LMzNDV0+dwPsyrO8sD9wsTeHwOBigzS+oFDr5MKakF7sULUDDLG/yxW4eQOWE5duvwBdUFrPwFudGewYMbbelDDjfaG5QUsFKE5bWFRwsMkBvNa2ZWIMOnEtygxmg0QOVIxbXCDOh7IFJf8qA4y2sLj1aY9906hLXzroKDZbcOXsuR8orBYEB+fj42b96MX/ziF4KvU7R4AQpmeYqUCxwAqMaNFrNiQlVVFX7yk5+gvLwc/f39qKurEzQGxYvXAq8CpgoOfG565ytiVkx4/vnncfLkSWzduhU3btzAtWvXBI2Bv2+8D/AYjQbUkZmlxAUOviBmxQQA2LNnDwDghRdeEFzuJGAsrwVyoz1HlQscGAMzdTkcclRM+Oqrr1BYWIjCwkLMmDEDaWlpgu7F37dcJHi0wmqaEwaUtcDBghwVE2JiYrzaHjbgLK8tarTCvMwJA/Lt1iEnVDFBZHgVMAWzpAtmeQtVTOAQHgUMqMMKy7X1rDcoqWKCoD2shFJdXY2cnBz86U9/wpYtWwRdEx09FSdONoo1BEHw9iwMSF/hXeo9s4SKzN2eWbbPyN7umfW7F5e43YdKM3Yiglf/zuF89MnXFbOHlWiWt6urC6+88ordNrC8wqMVVpMbPdwKC3GjAZGtMDMDpnsOh5JWFYkm3tLSUmzbtg1arVasLiWFglmeEQjBLCH4+szrT0QR77Fjx/DNN99g9erVYnTnV9QmYIAvKwzwFcxSkuUV9C1xt2/zq6++iuPHjwu6oV6vt5v47umRf2WHWueEAX4zswwdPejs7keyNtzjCg6+oKT1vD4HrM6cOYNnn30W4eFDbk9HRwdCQ0OxZcsWlJWVub1ejoDVSPAmYkB9wSyLtRUazBou4L0lS90HrMIjEbzsFYfz40/uQk5Ojk9JGv7C55+tp556Crdv37b+f8OGDZgzZ47gaDNvqNUK85KZJeY64RExm8Ee3HU4rSTLq5p5Xk/g8TkYUEcwS4rdOgIV0b8Nf/nLX8TuUhZotw7P4Wm3DjVAltcNPFphNe/WYRuNBlzv1uEtARdtVjtkhT1D7goOvqCkZ14SrwdQMMsz5ApmCYKZwR7cEdaWU8ht9hAe3WiAglnDg1lqgMTrBWpMrQSUs1uHL9Azr0ogN9oz5Nitw1OU9MxLltdHeLTAgDrcaMD1Agc1IOp6Xm8IDg7G5MmT5RyCUx48eICIiAi5h2EHjUkYX3/9Nfr6HMt32pKZmYmbN286nJ8yZQrq6+ulGpqoyC5ey24EvMHjuGhMhC3kNhOEQiHxEoRCCdq+fft2uQfxgx/8QO4hOIXHcdGYCAuyP/MSBOEd5DYThEIh8RKEQvG7eDds2IDY2FhkZGQgIyMD27Ztc9n2P//5DzIyMpCSkoKcnByn83Ji8Nprr2H69OnQ6XSYO3cuTp065bKtRqNBenq6dfynT58WdSxCa7vu378fycnJSExMRFFRkbW8htj09vZi5cqVSElJQUZGBpYuXYqrV686tKuurkZ4eLj1fcnIyIDJZJJkTMS3MD/z/PPPsz179rhtZzabWWJiIvvkk08YY4zt3LmTrVmzRpIxffjhh6ynp4cxxlhTUxObMGECM5lMTtsCYF1dXZKMgzHGFi9ezA4cOMAYY+zdd99lWVlZDm0uX77MpkyZwtrb25nZbGbLly9nFRUVkozHZDKxDz74gJnNZsYYY3v27GE//OEPHdp98sknbPbs2ZKMgXAOt25zQ0MDQkNDsWjRIgBDBYvff/999PeLX0gqNzcXYWFhAICZM2dicHAQHR0dot/HHZbaruvXrwcwVNv1ypUrDpbun//8J3784x8jOjoaGo0GxcXFOHLkiCRjGjNmDPLy8qyV77KysnD58mVJ7kV4hizi1ev1SE9Px7Jly9DU1OS0zfXr1+2qjo8bNw7jxo2TzHW2cODAASQmJiI2NtZlm0WLFkGn06G0tBTd3d2i3Vtobdfh701CQsKI9V/FZPfu3Vi+fLnT1y5evIhZs2YhMzMT5eXlfhmPmhE9e93dHs+///3vMWXKFIwaNQpHjx5Fbm4uDAaD0/xYT+ucejumuLg4AMDJkydRVlY24h7U165dQ3x8PLq7u1FcXIxt27aJ+kUV+jfbtvP2ffGUHTt2wGAwoKKiwuG1WbNmwWg0IjIyEkajEXl5edBqtSgoKPDL2FSJzG47S0lJYQ0NDQ7nz58/z2bMmGH9//3791loaCh7+PChJOOorq5mcXFxrKmpSfA1Z8+eZWlpaaKN4datW2z8+PGsv7+fMTb03B8dHc2uXLli1+6Pf/wj27x5s/X/H3zwAcvOzhZtHM7YuXMnmz17Nuvs7BTUfseOHWzLli2Sjknt+N1ttk1ir6urw507d+zKI1qYPXs2ent7UV1dDWCobOLKlSsREuJb5XNn1NbW4rnnnkNlZSV0Op3Ldp2dnejp6QEAmM1m/P3vf8cTTzwh2jiE1nZdtWoVjh49ilu3boExhoqKCqxZs0a0cQxHr9fjyJEjOH78OCZMmOC0zc2bN2E2mwEMFZ2rqqoS9b0hnODvX4ucnByWlpbGdDody8rKYqdOnbK+9uc//5n98pe/tP7/7NmzLD09nSUnJ7NFixYxo9EoyZiSkpJYVFQU0+l01uPChQsOYzp79iybOXMmS09PZzNmzGDr169nd+7cEXUsX3zxBcvKymLJycls9uzZrLm5mTHGWGFhIausrLS2e+utt1hiYiKbNm0aKywslMwjuXHjBgPAvve971nfm7lz5zqMac+ePWzGjBnW9+bXv/61NUJNSAOlRxKEQuF2qoggiJEh8RKEQiHxEoRCIfEShEIh8RKS8tJLLyEhIQEajQbNzc2i9l1eXo7p06cjLS0N6enp6O3tFbV/3iHxEpKyevVqnDlzxi6dUwwqKytx+PBh1NXVobm5GSdOnJAkB4BnSLyEpCxcuNBpnrjBYMAzzzyDzMxM6HQ6j1NMd+7cibKyMkRGRgIYSnAJCvK+ooMSIfESfmdwcBDr1q3Drl27UF9fj3PnzqGiogKNjY2C+2htbUVDQwPmz5+POXPmYPfu3RKOmE/43O6fCGguXryIlpYWu5TOrq4utLa2YtasWYIWkgwMDODSpUuora3FvXv3kJ2djaSkJOTl5fnrz5AdEi/hdxhj0Gq1LpeDCtmdJD4+HmvXrkVQUBAmTZqE3NxcnD9/XlXiJbeZ8DupqakIDw/HwYMHrefa2tpw9+5dwX2sW7cOH330EYChrXpqampGXFQSiFBuMyEpJSUlqKysRHt7O7RaLSIiItDW1gaDwYCXX34Z169fx+DgIB577DEcPnwYMTExgvo1mUzYtGkTGhoaoNFokJ+fDw62IPcrJF6CUCjkNhOEQiHxEoRCIfEShEIh8RKEQiHxEoRCIfEShEIh8RKEQiHxEoRCIfEShEL5PylYJg63Or+JAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 240x160 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "levelsToPlot = 10**(np.arange(2,20) / 20 * 4 - 9)\n",
    "if doublePrecision:\n",
    "    levelsToPlot = 10**(np.arange(-4,20) / 20 * 4 - 9)\n",
    "\n",
    "from matplotlib import ticker, cm\n",
    "from matplotlib.colors import LogNorm\n",
    "plt.figure(num=None, figsize=(3, 2), dpi=80, facecolor='w', edgecolor='k')\n",
    "plt.contourf(X, Y, Z, levels=levelsToPlot, locator=ticker.LogLocator(), cmap=cm.PuBu_r)\n",
    "\n",
    "\n",
    "cbar = plt.colorbar()\n",
    "cbar_ticks = 10.0**(np.arange(0,15)-9)\n",
    "cbar.set_ticks(cbar_ticks)\n",
    "plt.savefig('bifurcationZone'+bitsString+'.pdf', bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "65"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 0.]], dtype=torch.float32)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}