training-phases-plots/constant-weights-float16-vs-float32-tf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "RJqZjpyIx7Qb",
    "outputId": "977b8694-da94-43dc-d0cc-a1b57716f078"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: deepxde in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (1.9.3)\n",
      "Requirement already satisfied: matplotlib in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from deepxde) (3.8.0)\n",
      "Requirement already satisfied: numpy in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from deepxde) (1.26.0)\n",
      "Requirement already satisfied: scikit-learn in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from deepxde) (1.3.1)\n",
      "Requirement already satisfied: scikit-optimize>=0.9.0 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from deepxde) (0.9.0)\n",
      "Requirement already satisfied: scipy in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from deepxde) (1.11.3)\n",
      "Requirement already satisfied: joblib>=0.11 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from scikit-optimize>=0.9.0->deepxde) (1.3.2)\n",
      "Requirement already satisfied: pyaml>=16.9 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from scikit-optimize>=0.9.0->deepxde) (23.9.7)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from scikit-learn->deepxde) (3.2.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from matplotlib->deepxde) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from matplotlib->deepxde) (0.12.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from matplotlib->deepxde) (4.43.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from matplotlib->deepxde) (1.4.5)\n",
      "Requirement already satisfied: packaging>=20.0 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from matplotlib->deepxde) (23.2)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from matplotlib->deepxde) (10.0.1)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from matplotlib->deepxde) (3.1.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from matplotlib->deepxde) (2.8.2)\n",
      "Requirement already satisfied: PyYAML in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from pyaml>=16.9->scikit-optimize>=0.9.0->deepxde) (6.0.1)\n",
      "Requirement already satisfied: six>=1.5 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from python-dateutil>=2.7->matplotlib->deepxde) (1.16.0)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Requirement already satisfied: matplotlib in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (3.8.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from matplotlib) (1.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from matplotlib) (0.12.0)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from matplotlib) (4.43.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from matplotlib) (1.4.5)\n",
      "Requirement already satisfied: numpy<2,>=1.21 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from matplotlib) (1.26.0)\n",
      "Requirement already satisfied: packaging>=20.0 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from matplotlib) (23.2)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from matplotlib) (10.0.1)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from matplotlib) (3.1.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from matplotlib) (2.8.2)\n",
      "Requirement already satisfied: six>=1.5 in /Users/Jeniffer/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.2.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install deepxde\n",
    "!pip install matplotlib"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "rd7uxjWKydXN",
    "outputId": "4a85e477-c6e7-45fb-f08f-ff402b615004"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: DDE_BACKEND=tensorflow\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-03-27 22:40:33.522236: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "Using backend: tensorflow\n",
      "Other supported backends: tensorflow.compat.v1, pytorch, jax, paddle.\n",
      "paddle supports more examples now and is recommended.\n"
     ]
    }
   ],
   "source": [
    "%env DDE_BACKEND=tensorflow\n",
    "import deepxde as dde\n",
    "import numpy as np\n",
    "from deepxde.backend import tf\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def func(x):\n",
    "    return x * np.sin(5 * x)\n",
    "\n",
    "def get_weights(denses):\n",
    "    weights = np.concatenate([layer.get_weights()[0].flatten() for layer in denses])\n",
    "    biases = np.concatenate([layer.get_weights()[1] for layer in denses])\n",
    "    return weights\n",
    "\n",
    "def train(model, iterations, display_every, loss, constant, norm_weights):\n",
    "    prev_weight = None\n",
    "    for i in range(iterations):\n",
    "        model.train_state.set_data_train(\n",
    "            *model.data.train_next_batch(model.batch_size)\n",
    "        )\n",
    "        model.train_step(\n",
    "            model.train_state.X_train,\n",
    "            model.train_state.y_train,\n",
    "            model.train_state.train_aux_vars,\n",
    "        )\n",
    "\n",
    "        X_test, y_test = model.data.test()\n",
    "        y_pred = model.predict(X_test)\n",
    "        l2r = np.linalg.norm(y_pred - y_test) / np.linalg.norm(y_test)\n",
    "        loss.append(l2r)\n",
    "\n",
    "        w = get_weights(model.net.denses)\n",
    "        norm_weights.append(np.linalg.norm(w))\n",
    "        if model.train_state.epoch != 0:\n",
    "            num_const = 0\n",
    "            for i in range(len(w)):\n",
    "                if w[i] == prev_weight[i]:\n",
    "                    num_const += 1\n",
    "            constant.append(100*num_const/len(w))\n",
    "        prev_weight = w\n",
    "\n",
    "        model.train_state.epoch += 1\n",
    "        model.train_state.step += 1\n",
    "        if model.train_state.step % display_every == 0 or i + 1 == iterations:\n",
    "            print(str(model.train_state.step) + \" \" + str(loss[-1]))\n",
    "\n",
    "def test(seed, plot=True):\n",
    "    dde.config.set_default_float(\"float32\")\n",
    "    dde.config.set_random_seed(seed)\n",
    "    print(\"Training Float32:\")\n",
    "    geom = dde.geometry.Interval(-1, 1)\n",
    "    num_train = 16\n",
    "    num_test = 100\n",
    "    data = dde.data.Function(geom, func, num_train, num_test)\n",
    "\n",
    "    activation = \"tanh\"\n",
    "    initializer = \"Glorot uniform\"\n",
    "    net = dde.nn.FNN([1] + [10] * 2 + [1], activation, kernel_initializer =  tf.keras.initializers.glorot_uniform(seed=seed))\n",
    "\n",
    "    model = dde.Model(data, net)\n",
    "    model.compile(\"adam\", lr=0.001, metrics=[\"l2 relative error\"])\n",
    "    loss_32 = []\n",
    "    constant_32 = [0]\n",
    "    norm_32 = []\n",
    "    train(model, 10000, 1000, loss_32, constant_32, norm_32) # 10000\n",
    "    print([constant_32[1]] + [constant_32[10]] + [constant_32[100]] + [constant_32[1000]] + [constant_32[9999]] )\n",
    "\n",
    "    dde.config.set_default_float(\"float16\")\n",
    "    dde.config.set_random_seed(seed)\n",
    "    print(\"Training Float16:\")\n",
    "    geom = dde.geometry.Interval(-1, 1)\n",
    "    num_train = 16\n",
    "    num_test = 100\n",
    "    data = dde.data.Function(geom, func, num_train, num_test)\n",
    "\n",
    "    activation = \"tanh\"\n",
    "    initializer = \"Glorot uniform\"\n",
    "    net = dde.nn.FNN([1] + [10] * 2 + [1], activation, kernel_initializer =  tf.keras.initializers.glorot_uniform(seed=seed))\n",
    "\n",
    "    model = dde.Model(data, net)\n",
    "    model.compile(\"adam\", lr=0.001, metrics=[\"l2 relative error\"])\n",
    "    loss_16 = []\n",
    "    constant_16 = [0]\n",
    "    norm_16 = []\n",
    "    train(model, 10000, 1000, loss_16, constant_16, norm_16) # 10000\n",
    "    # print([constant_16[1]] + [constant_16[10]] + [constant_16[100]] + [constant_16[1000]] + [constant_16[9999]] )\n",
    "\n",
    "    if plot:\n",
    "        x = [i for i in range(len(loss_32))]\n",
    "        plt.figure(figsize=(4.8,3.6))\n",
    "        plt.ylabel('Percentage of constant weights')\n",
    "        plt.xlabel('No. of iterations')\n",
    "        plt.plot(x, constant_32, label = \"Float32\", color = 'red')\n",
    "        plt.plot(x, constant_16, label = \"Float16\", color = 'orange')\n",
    "        leg1 = plt.legend(loc = 'right', frameon=False)\n",
    "        plt.savefig(\"constant-weights-\" + str(seed) + \".pdf\", bbox_inches='tight')\n",
    "        plt.show()\n",
    "    return (constant_32, constant_16, norm_32, norm_16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "MCC1BRklySWe",
    "outputId": "4d0b3321-43ad-4001-e6f7-bddc3a5e78dd"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Set the default float type to float32\n",
      "Training Float32:\n",
      "Compiling model...\n",
      "'compile' took 0.002713 s\n",
      "\n",
      "1000 0.42378744\n",
      "2000 0.27962446\n",
      "3000 0.047656193\n",
      "4000 0.039519608\n",
      "5000 0.033346634\n",
      "6000 0.026610268\n",
      "7000 0.020320777\n",
      "8000 0.017603723\n",
      "9000 0.014633729\n",
      "10000 0.012508991\n",
      "[0.0, 0.0, 0.0, 0.0, 0.0]\n",
      "Set the default float type to float16\n",
      "Training Float16:\n",
      "Compiling model...\n",
      "'compile' took 0.003074 s\n",
      "\n",
      "1000 0.3599\n",
      "2000 0.168\n",
      "3000 0.0795\n",
      "4000 0.0686\n",
      "5000 0.06476\n",
      "6000 0.06323\n",
      "7000 0.062\n",
      "8000 0.06064\n",
      "9000 0.06085\n",
      "10000 0.0608\n",
      "Set the default float type to float32\n",
      "Training Float32:\n",
      "Compiling model...\n",
      "'compile' took 0.002237 s\n",
      "\n",
      "1000 0.42936796\n",
      "2000 0.0504884\n",
      "3000 0.036177512\n",
      "4000 0.02719491\n",
      "5000 0.02487871\n",
      "6000 0.02346393\n",
      "7000 0.021664754\n",
      "8000 0.01979175\n",
      "9000 0.018077174\n",
      "10000 0.016513951\n",
      "[0.0, 0.0, 0.0, 0.0, 0.8333333333333334]\n",
      "Set the default float type to float16\n",
      "Training Float16:\n",
      "Compiling model...\n",
      "'compile' took 0.002158 s\n",
      "\n",
      "1000 0.458\n",
      "2000 0.4307\n",
      "3000 0.401\n",
      "4000 0.3364\n",
      "5000 0.0944\n",
      "6000 0.0768\n",
      "7000 0.0726\n",
      "8000 0.0696\n",
      "9000 0.06726\n",
      "10000 0.0663\n",
      "Set the default float type to float32\n",
      "Training Float32:\n",
      "Compiling model...\n",
      "'compile' took 0.001837 s\n",
      "\n",
      "1000 0.41661754\n",
      "2000 0.16488421\n",
      "3000 0.054840226\n",
      "4000 0.04012658\n",
      "5000 0.027988333\n",
      "6000 0.023002403\n",
      "7000 0.02129294\n",
      "8000 0.02026394\n",
      "9000 0.019567886\n",
      "10000 0.017958723\n",
      "[0.0, 0.0, 0.0, 0.0, 0.0]\n",
      "Set the default float type to float16\n",
      "Training Float16:\n",
      "Compiling model...\n",
      "'compile' took 0.002055 s\n",
      "\n",
      "1000 0.2544\n",
      "2000 0.131\n",
      "3000 0.1094\n",
      "4000 0.09827\n",
      "5000 0.089\n",
      "6000 0.08307\n",
      "7000 0.0799\n",
      "8000 0.07904\n",
      "9000 0.0788\n",
      "10000 0.0788\n",
      "Set the default float type to float32\n",
      "Training Float32:\n",
      "Compiling model...\n",
      "'compile' took 0.001847 s\n",
      "\n",
      "1000 0.4181815\n",
      "2000 0.08287758\n",
      "3000 0.06660479\n",
      "4000 0.05314068\n",
      "5000 0.032243434\n",
      "6000 0.019560345\n",
      "7000 0.017836768\n",
      "8000 0.017154194\n",
      "9000 0.016094703\n",
      "10000 0.01608537\n",
      "[0.0, 0.0, 0.0, 0.0, 0.0]\n",
      "Set the default float type to float16\n",
      "Training Float16:\n",
      "Compiling model...\n",
      "'compile' took 0.002554 s\n",
      "\n",
      "1000 0.10657\n",
      "2000 0.0944\n",
      "3000 0.0883\n",
      "4000 0.0845\n",
      "5000 0.079\n",
      "6000 0.07513\n",
      "7000 0.07135\n",
      "8000 0.0695\n",
      "9000 0.0678\n",
      "10000 0.06696\n",
      "Set the default float type to float32\n",
      "Training Float32:\n",
      "Compiling model...\n",
      "'compile' took 0.002591 s\n",
      "\n",
      "1000 0.17296779\n",
      "2000 0.07221565\n",
      "3000 0.04422923\n",
      "4000 0.022260169\n",
      "5000 0.017263152\n",
      "6000 0.015492117\n",
      "7000 0.014048989\n",
      "8000 0.012760271\n",
      "9000 0.011500776\n",
      "10000 0.010601249\n",
      "[0.0, 0.0, 0.0, 0.0, 0.0]\n",
      "Set the default float type to float16\n",
      "Training Float16:\n",
      "Compiling model...\n",
      "'compile' took 0.001822 s\n",
      "\n",
      "1000 0.3672\n",
      "2000 0.3\n",
      "3000 0.2148\n",
      "4000 0.1735\n",
      "5000 0.1566\n",
      "6000 0.1396\n",
      "7000 0.1123\n",
      "8000 0.0978\n",
      "9000 0.0953\n",
      "10000 0.094\n",
      "Set the default float type to float32\n",
      "Training Float32:\n",
      "Compiling model...\n",
      "'compile' took 0.001803 s\n",
      "\n",
      "1000 0.40494394\n",
      "2000 0.03849126\n",
      "3000 0.026462758\n",
      "4000 0.024365164\n",
      "5000 0.022622045\n",
      "6000 0.020518444\n",
      "7000 0.019042527\n",
      "8000 0.01841931\n",
      "9000 0.017490564\n",
      "10000 0.0154223265\n",
      "[0.0, 0.0, 0.0, 0.0, 0.8333333333333334]\n",
      "Set the default float type to float16\n",
      "Training Float16:\n",
      "Compiling model...\n",
      "'compile' took 0.002138 s\n",
      "\n",
      "1000 0.2146\n",
      "2000 0.094\n",
      "3000 0.06757\n",
      "4000 0.0595\n",
      "5000 0.05618\n",
      "6000 0.05972\n",
      "7000 0.0646\n",
      "8000 0.0687\n",
      "9000 0.0791\n",
      "10000 0.0726\n",
      "Set the default float type to float32\n",
      "Training Float32:\n",
      "Compiling model...\n",
      "'compile' took 0.002446 s\n",
      "\n",
      "1000 0.4370987\n",
      "2000 0.16317655\n",
      "3000 0.035300013\n",
      "4000 0.031973448\n",
      "5000 0.02821659\n",
      "6000 0.024476402\n",
      "7000 0.022830939\n",
      "8000 0.021657275\n",
      "9000 0.020278946\n",
      "10000 0.01868041\n",
      "[0.0, 0.0, 0.0, 0.0, 0.0]\n",
      "Set the default float type to float16\n",
      "Training Float16:\n",
      "Compiling model...\n",
      "'compile' took 0.001817 s\n",
      "\n",
      "1000 0.21\n",
      "2000 0.1112\n",
      "3000 0.10767\n",
      "4000 0.0987\n",
      "5000 0.0909\n",
      "6000 0.08514\n",
      "7000 0.08185\n",
      "8000 0.08105\n",
      "9000 0.08026\n",
      "10000 0.0781\n",
      "Set the default float type to float32\n",
      "Training Float32:\n",
      "Compiling model...\n",
      "'compile' took 0.002459 s\n",
      "\n",
      "1000 0.38877404\n",
      "2000 0.04495946\n",
      "3000 0.02833704\n",
      "4000 0.023567982\n",
      "5000 0.02067688\n",
      "6000 0.018415727\n",
      "7000 0.015072536\n",
      "8000 0.012132415\n",
      "9000 0.011388482\n",
      "10000 0.011028597\n",
      "[0.0, 0.0, 0.0, 0.0, 0.0]\n",
      "Set the default float type to float16\n",
      "Training Float16:\n",
      "Compiling model...\n",
      "'compile' took 0.001774 s\n",
      "\n",
      "1000 0.3518\n",
      "2000 0.2847\n",
      "3000 0.2532\n",
      "4000 0.234\n",
      "5000 0.2034\n",
      "6000 0.1763\n",
      "7000 0.1558\n",
      "8000 0.1469\n",
      "9000 0.1404\n",
      "10000 0.1497\n",
      "Set the default float type to float32\n",
      "Training Float32:\n",
      "Compiling model...\n",
      "'compile' took 0.002063 s\n",
      "\n",
      "1000 0.334095\n",
      "2000 0.02597841\n",
      "3000 0.020961605\n",
      "4000 0.014602997\n",
      "5000 0.011188525\n",
      "6000 0.010178124\n",
      "7000 0.009218896\n",
      "8000 0.008829812\n",
      "9000 0.008613834\n",
      "10000 0.008515765\n",
      "[0.0, 0.0, 0.0, 0.0, 0.0]\n",
      "Set the default float type to float16\n",
      "Training Float16:\n",
      "Compiling model...\n",
      "'compile' took 0.002559 s\n",
      "\n",
      "1000 0.3254\n",
      "2000 0.2366\n",
      "3000 0.1624\n",
      "4000 0.1209\n",
      "5000 0.10504\n",
      "6000 0.09656\n",
      "7000 0.09204\n",
      "8000 0.08966\n",
      "9000 0.0882\n",
      "10000 0.08704\n",
      "Set the default float type to float32\n",
      "Training Float32:\n",
      "Compiling model...\n",
      "'compile' took 0.001968 s\n",
      "\n",
      "1000 0.38732782\n",
      "2000 0.03663202\n",
      "3000 0.024344534\n",
      "4000 0.021882944\n",
      "5000 0.021489482\n",
      "6000 0.021251125\n",
      "7000 0.021034122\n",
      "8000 0.020802652\n",
      "9000 0.020526523\n",
      "10000 0.020290682\n",
      "[0.0, 0.0, 0.0, 0.0, 0.0]\n",
      "Set the default float type to float16\n",
      "Training Float16:\n",
      "Compiling model...\n",
      "'compile' took 0.002472 s\n",
      "\n",
      "1000 0.3967\n",
      "2000 0.1838\n",
      "3000 0.05673\n",
      "4000 0.05524\n",
      "5000 0.05383\n",
      "6000 0.05328\n",
      "7000 0.05276\n",
      "8000 0.05234\n",
      "9000 0.05234\n",
      "10000 0.05203\n"
     ]
    }
   ],
   "source": [
    "trials = 10\n",
    "const_32 = np.zeros((trials, 10000))\n",
    "const_16 = np.zeros((trials, 10000))\n",
    "norm_32 = np.zeros((trials, 10000))\n",
    "norm_16 = np.zeros((trials, 10000))\n",
    "for i in range(trials):\n",
    "    const_32[i], const_16[i], norm_32[i], norm_16[i] = test(i, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 392
    },
    "id": "4ZIn469w2_aQ",
    "outputId": "266649ee-741a-481c-fe05-2521347b00b0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000,)\n",
      "(10000,)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWsAAAEPCAYAAACEI+U0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABrjklEQVR4nO2deXxMd/fHPzOTzGTfJZFIJGoLYt9iC6WlVKP1VGnaB1Xap7SU6sOvVVuJerqglKrnQVtqae2UEluRxr7vxFKSIJFdtpnv74/jO/dOMom5yWT1fb9e88rMnbucO5n53HPP93zPUTHGGBTy008/4fvvv8f169cRExOD2rVrY86cOQgODkZERITS3QkEAoHgCaiVbrBw4UKMHTsWL7zwAlJSUqDX6wEAbm5umDNnjrXtEwgEAgFKINbffvstfvjhB3zyySfQaDTG5a1bt8aZM2esapxAIBAICMViHRcXhxYtWhRartPpkJmZaRWjBAKBQGCKYrEODg7GyZMnCy3fvn07QkJCrGGTQCAQCApgo3SDsWPHYuTIkcjOzgZjDIcPH8Yvv/yCqKgoLFmypCxsFAgEgqceVUmyQVasWIEpU6bg2rVrAAA/Pz9MnToVw4YNs7qBAoFAICihWHOysrKQkZEBb29va9okEAgEggKUSqwFAoFAUD4ojlkHBwdDpVIV+f7169dLZVBFYDAYcPfuXTg7Oxd7bgKBQKAExhjS09Ph5+cHtVpxPocJisV6zJgxJq/z8vJw4sQJbN++HePHjy+VMRXF3bt3ERAQUNFmCASCasrt27dRq1atUu1DsViPHj3a7PIFCxbg6NGjpTKmonB2dgZAH6iLi0sFWyMQCKoLaWlpCAgIMGpMabBazPr69eto3rw50tLSrLG7ciUtLQ2urq5ITU0VYi0QCKyGNbWldEEUGb/++is8PDystTuBQCAQyFAcBmnRooXJIBxjDAkJCbh//z6+++47qxonEAgEAkKxZ92vXz9EREQYH6+88gomT56Ms2fPYsSIEYr2tX//fvTt2xd+fn5QqVTYsGGDyfuMMXz22WeoWbMm7O3t0aNHD1y5csVkneTkZERGRsLFxQVubm4YNmwYMjIylJ6WQCAQVGoUe9aTJ0+22sEzMzPRrFkzvPXWW3jllVcKvT979mzMmzcPy5cvR3BwMCZNmoSePXvi/PnzsLOzAwBERkYiPj4eO3fuRF5eHoYOHYoRI0Zg5cqVVrNTIBAIKhqLBhiVDBqWNIiuUqmwfv169OvXDwB51X5+fhg3bhw++ugjAEBqaip8fHywbNkyDBw4EBcuXECjRo1w5MgRtG7dGgAVlOrduzf+/vtv+Pn5WXRsMcAoEAjKAmtqi0WetZub2xMnizDGoFKpjM0ISktcXBwSEhLQo0cP4zJXV1e0a9cOMTExGDhwIGJiYuDm5mYUagDo0aMH1Go1YmNj8fLLL5vdd05ODnJycoyvq2IGi0AgeLqwSKz37NlT1nYUIiEhAQDg4+NjstzHx8f4XkJCQqG6JDY2NvDw8DCuY46oqChMnTrVyhYLyoNDhwDGgOTkwu/l5QG5ufQ+x8YGyM8HVCqATyCztQWys2mZjQ0t59totdK23boBjo60HicjA7Czo+2UcuAAEBwM+PsXvc6RI7T/Jk1MjwsAV68C8fFA587SstOngZs36TljwPPPAzt20OuCE+acnMj+gtjZ0efxJPz9gTt3pM+KscI2ArRMrwd4bxLGpAd/PyeH/g8qFf3PNBraxs6O1tPr6f8J0HvyY+bnS58/36+NDW1jMEjnzp9z+DZqtWS3Wk0P/h3hxzEYJJv4dyg/n74PvXqR7eWNRV+58PDwsrajXJk4cSLGjh1rfM0T1wWVBy6mWVkkcgDQrBlw/z4tz8+XfmT8R52QADx8KO2DMaBWLRIYgNa3sQG8vIC7d2k/Gg1gby9t88wzktAU9FECA4Fbt4AaNYD27ZWdj8EAPHhA9tWsCaSkkD0uLvT39Gl6Hh9Pdt24Qdv17EkXkLQ04MIFOqfNm+m9zp1JqPV6SSB37qTPzMaGHlxQs7JMhY5vo1YDjx5J6+XnS/ZqtbQe/zzu3JHE3mCg5Wq1JGj29rQ8M5PE2NkZ0Olon9nZtC0XygcPSJjVaiA9ndbLyQF8fOhYDx/ScpWK1svJkf7m50sXArmt+flkC/9fZ2fThYDPR+EX5dRUwNubvgM6HbBvH32OGRm0zWef0XabNwPnztG2L7wA1K4NNG9O61RasTZHVlYWbt26hdzcXJPlTZs2LbVRAODr6wsASExMRM2aNY3LExMT0bx5c+M69+7dM9kuPz8fycnJxu3NodPpoNPprGKnoGz44w9JgAwGen7yJAmzXk8Cw0UkLY2ep6bSj7FGDRI+Dw/arkYN2sepU/SXr8cYveYemEpF+0pLAxwcAE9PWsY9rlu36Dj375Mg2dpKnnhuLr0nF345/DgaDRAbS2JlMEjer9zzBCTPk3vJ8n3wbf78k/7KL1wqFYmJVisJal4enTP3oO3syF6DgY7BxZsxEklbW9rGxYU+55wc8srt7OiOxsZGEnh+7mo1CW1ODpCUJF0YuMByIeTnajDQcr6P/Hx6fveuqT0GA3D5Mnm0Tk5kW/Pm9F3IyaHz5J66SgW0aAEsX04XvSNHaN+LFwNubnTxXbjwyd+9adMKL/v9d/o7ceKTty8rFIv1/fv3MXToUPzOrS+AtWLWwcHB8PX1RXR0tFGc09LSEBsbi3/9618AgLCwMKSkpODYsWNo1aoVAGD37t0wGAxo166dVewQKIeLK/fWLK2NtXMn/aD5bXBWFu0rKYl+rO7uJBb5+SSW+fn0Y83OljzAtm0L75d7QQ0bAhcvktA4OwP16pFtx49LopGURMfNyCCvizESWJWKjsXX272b/jJGF4WHD4sOCwB0V8DF9sEDsjUzky4K/DPjdxN6PZ2njw/ZrtGQePLPRaORbFKryWadjtbhIR65J52XR+fs4kIevYODJJQqFX0W+fl0ztybBWjdvDx6LyuLtrl6FahTh5ZrNKYXO7Va+v9otbSOTgfcvk3vu7tLn9mFC8C6dfT62WelzxMAgoJou0uXiv6ujBhBIgwAffvSnQkATJ9eeN2FC4EpUywT6icRFVVxgl2iQk4pKSmIjY1F165dsX79eiQmJuLzzz/HV199pWhfGRkZuHr1qvF1XFwcTp48CQ8PDwQGBmLMmDH4/PPPUa9ePWPqnp+fnzFjJCQkBL169cLw4cOxaNEi5OXlYdSoURg4cKDFmSBPM6mp9OO39sTTrVtJaBITpWV16tCPMDWVPNeGDelHu2UL3ZI2akRCwT0ug4F+5FxgcnJIDB4+lN6XrwtQCKM4XFwoFnzuHB2fi1JwMH0ON29KF4G8PBJBjk5H63Bv08VFEliA1ue347m55B1qNFJY4fhx6a5ApSJhvH+fvH7uRaelSQL76BEtc3Ki7VNSaJmTkxRKsLOjdR48kMIRdnZ03KwsyVvln09yMh03I0Oyi2+XmkphpLt3gb//pv27u0sXh/btgc8/lz6P556jiysA9OgB3LtHf7/+mpa5u9P/ytNT+hyHDgWWLi38f5ELNSCFgIqDCzUghYWK4sQJICLiyfus7CiuDVKzZk1s3LgRbdu2hYuLC44ePYr69etj06ZNmD17Ng7wAKMF7N27F926dSu0fPDgwVi2bBkYY5g8eTIWL16MlJQUdOrUCd999x3q169vXDc5ORmjRo3C5s2boVar0b9/f8ybNw9O/FtuAU9j6t79+8Bff9GPOTgYCA2lH/iRI0BphigyM4Fdu0gE8vKkOKF8sAmQluflSR5rejrZxUMCDx9Kgzs85MBvv+WDVf7+FH7gnltJ+ftvun3m+7a1NR2U5PHP3Fw6Zn4+CaC3N4m2wUBea1oana+jIwkjv7vg58G3e/SI1udeKr874OJqZ0f70OslIecDcPn5dM4ODtJnxuH7YowumHFxQN26FCvPzyfx3LKFhL93bwrL8JCK4MkkJ1v+XbOmtigWaxcXF5w+fRpBQUGoXbs2Vq5ciY4dOyIuLg6NGzdGVlZWqQyqCJ5Gsd682XTAqVUrKRzQpQvF+CxBrwe2bQM6daKYcHo6CZeNjXRLzUMIAF0QNBoSPT4Q5epKx71/n7w77vFyLzQoiASHj/BrNHRx4QNo8sGm0sK9wKwsabAPkC4aXAg9PWkdfquvUpFt9vbSBcXOjtaRZ5/wGC4X/Ph4aVDMYCCvsmtXICaGxJUPLubkACtW0ICpTkcXVEdHsou/d/MmZbB07Aj873/kJQuKZsAAutgtW0avg4Is8+ofPrT891HuedZyGjRogEuXLiEoKAjNmjXD999/j6CgICxatMhkIFBQueGDT+npJJZcqPV68sYs+TImJJAnrtdTxobBQOKUnEyidO+edBvv6CgNNGm1JDJJSdLteF4e3erL061CQii26eBAMd9Tp+gHVZZd5Dw9pb8GgxTK4SEFbm9aGgkujzFzl0celuHxXj6Yx5fduwccO0Y/+hs3aFBMXl24uMm3KSn099gxKYNCzp49FIISQl08fCJ2jRr0P75wARg/ni6eu3YBbdpQlgiPhcspZQ+BEqPYs/7555+Rn5+PIUOG4NixY+jVqxeSk5Oh1WqxbNkyvPbaa2Vla5nxtHnW+fnAhg0kIklJ5MHZ2pLo8Nvrvn2L3v7KFRqo46lbaWlSDm9qqpRex1O6ACmfmcdiXVykLA6emJOVRR7z40oCZu0uSX5zSWFMurjwuDJjwNmzdH4+PnTB4l50ZiYJ8NattN0779CP/8cfaX9vvgn89FP52S8omm++oe8dd0r44KxeTxdTgO7+NBr6X8rL+KelSemAT6JCwyAFycrKwsWLFxEYGAgvL69SGVNRVDexzssj4SjKO968mUSFx019fSXPV6UiT7tdO/I2/P0p3gmQWP7+uyTq9+/TFz4lhY6Vlkb745Ew+SQDnrnAQxs8PYxnJKjVlMdamb9Cej2dq7MzhRyeeYYuXFW0QdJTS0QE0KcPxZ35BZhfjA0GEmueJQPQ93rkSGn7ihJrxX7KgQMH0KlTJ+NrBwcHtGzZslRGCKxLbCyJce/eheO5yclSOhwftEpJkWZo2dqSWMfG0ntpaeTNnjlD2+fnU/YBz6flsdrsbCnb4ZlnpHAC3yYxkUIu9erRLTr3SNu2pVt6vb5yCzUAfPghhS3atgUOH65oa8qHJk3oToIzZIgU47UG/fsDv/1Gz1u1ou8Rn4gCABMmmE5OsQYvvUR3QjxHGzCdRGRvT78bOzv6Tpd24NpaKBbrZ599Fv7+/hg0aBDeeOMNNGrUqCzsEpQQnkVhMFActGC6+cGDJJ7p6VL8NT1diqs6OkpZHPyLfOaMlKWQmUlinZsrbQ9IKWwNGxb2OmxsTKdYBwSQjTzH+HGKfKVEr6cwxvr10rLqLtQjR1JYKzGRBjRfe41m9YWH0//KnFjL4+ctW9IgaVoasGYNzbTcupXe46mNAPD225SyGRJCy21tKTNp1SoK0wH0HRkxgr4vJ09Ky0vK+++TGNvZ0fe7Wzc69tat0mA4LzFgYyNNHPrqK8qg6du38DT28kKxWN+9exerVq3CL7/8glmzZqFp06aIjIzEoEGDSt0QUlB64uLorzz2xjl3Tppum5dH4Q8nJ+DaNdOJDTxjQ6ej5TodiXRmJm2bk0P7r1ePtn/0iLxsvV4aUHwSVproWmbExZnGKasroaHSXZO3N4mZRiOFpdLTSby+/JLWcXIC/u//gJkzTfczahTdIbVtS3dIBgMJ/Sef0P4cHGi6eq9eJJbXr9OxASktU62mvz170jYBAbSdTkfi2qUL0KED8PHHlp1by5Y0cM4ZM4bOUaMhIVap6Hzks0L9/SnExYWbj7U88wzw4otSuLAiUCzWXl5eGDVqFEaNGoW4uDisXLkSy5cvx8SJE9GlSxfsLpjhLihX0tOl/GU+cy87W5rAkJZG+cQA/RhUKpqVxtPHcnLIo3J0pB9VXp40seHRIxJsxig7gw8M8gHD0vLwIXmwvr7kmUVFUQbI0KHk0fDBxfR0sqOYigJFotdTfnmjRoVvb+/fB4YNK/VpWMTrr1O646+/SpNCnJxIqAICaGxAPoNv4EDyOAESUV68yRLGjQPmzSOvuHdv+p+uX0//w3r16JgnTlD9C35HpdNROEytJqF0caHvhk5H073Hj6f/QVISTaqqWRN44w36jhgMUnEl7sH26UMhJJ1OyrTh3x+djo7D89odHUmU8/MlG3Q65fU4CoYAvbwk8X3uOWl/8pmc3t7SjEs+F0Be9MnS2bhlQanG1oODgzFhwgQ0a9YMkyZNwr59+6xll6AEnDxJXzSeQufgQB7i2bN065mdTd52fj79UPkXj0/RvnBBmnKcm0v7yM2Vcn15+pq5Kd2l5eOPKcOEs2iR9HzpUnqsWEEi9vPPtPyNN6Tns2eTMAQFFT/t+9dfaT9eXpSLfOUKfU6PKxhYBWdn+gyL4scfKZR09y6JUs2a9Nn36iUJTEQEfQY5OSTqISFAv370f+jalf6P8+ebhhU6dqTxh3Pn6AKXkkLi06ABxX61WmkSzuDB0qxMR0eK49aoQQJsMNBnqdOZViHkwqtSkaeZn08izi/gPAaclycNGvMcdO7N2tmRE2BjQ+trNNKUeu7R86JQeXlkr3xS1JPSIdq1o/EWc9jbm87+LIhWK80Q5cgFv8qK9cGDB7FixQr8+uuvyM7ORkREBKKioqxpm0AB8fFUaAiQpg+7u5NQ8xBGejqJMWOSVyMnJITisbwuBy9OxAcUAfrhW4uEBMppzckxFeqiiIw0fc2FGij+1tjODliyhM55xQpa9uAB8NZb9Lc0jBpFn2nTpiS69+5J2QS8aBJjwIwZ0ja8Mx2/zW7QgMSP35Lzu5mJE2kfPLTUuTOdQ0YG3Rl88QWJ9+HDdAfy4osknjdu0DaOjnR8Nzf636vVJFi8ABIfVOPHtLUlcWRMElB5+Vh+EeReMS8CxUMYXHB5iIF75Xx7Ltb8uHx7PkWew0NwPCzCLwQFPWsvr8L/v9696X/RurXplHRA+s4XFFx+AapRgz6rrl0lLz8riy6+27ZVrFADJRDriRMnYtWqVbh79y6ee+45zJ07FxEREXDgo0WCcocPJvIfXX4+/SDT0+mHkJREr3le8+O6WGZp2ZJ+LHyyC88IASgz4En/Zh5G4bVBAgNpeXmGGAqSnU1eeEFKK9Rt2tAP++RJyrLhxZy4ePHsAjnPPCNVxNPp6IKamUkXRhcXKd3Rzk6afu7qKokf9+6cnCShffFFCjPY2dG6qam0bwcHSeScnaWUSS6WKhWtw+/E5OLIQ07cHi7aXLC5cPJp+Hy/cjsBqeAUdxDMedM6nVRpD5A+Oz5blXvo8rrWAHnRfOASkAYLn3mGLkoBARRH5/BzKyi6ajWFgPg5ywfI5XcW8vOqCBSL9f79+zF+/HgMGDCgyuZVVzf4j5rXDc7KotghL3fJp1Dr9SS48i9gQfgXlt/ehoTQ7XSNGk+OGd65Y91wQmWkbVu6CN28SWJw6pTkmdnZSUWeeKU8+eAVQDnrHh70v3JwINHlNU943J8LnqMj/eXT8Xk1Q0DyEl1dpYFgW1t6bm8viTX3ePlFlseGuYDyiSC8tjRHXi1RpaJz5bFcwFSYuZfMt+fLucjLt+HryS88/IIiLy4lLyXAvW6epz9lCg2KN2liKta9eknHU6vJy+bZI02aSKl45rBkslVFCjVQArE+ePBgWdghKAU3btCPndeQyMw0DWXwGCIfXbeEhg2l5+a2yc6m+O+ePTTQ5+xMXVyqC4sWAe++S4NyH35Ionr5Mg3uXb9OoQx/f6mBQG6uVMLVzY3Ekhe/t7Wl7In9+4GXX5ZEWB4K0GikmC8XJS768iYAPLTAxTIvj2zjVQm5GHMvlmf5yAUaMC1hy7cBTAVX7oGGhpKY8nop8hKt3FPn28kzi+T74OfJPXN+DLlXzreTN06Q70ejoXBTcLBpt6B69aT9y73/xYtpcL1JE0n0SxrO4Lakp5dtyYOiKMfJu4KyIC6O4r15eVItYe498JrGgYEkEJbOurLkmPK0tvv3rbPfkBAa6MrOppzeRYtoINTFBZg0yTrHsIRvvwX8/IBNm0yXN2xIYpWVRULNxdXNjYTUwUHKKuDCyQfYGjcmkXFwkGbN8Vt8QAoPAKbL5LfeXGS4d8oFiceJ5QN0PCbNvVUuhlzwAclTl4uv/Dicpk3p/Vat6Nx375aOz+Pu/HlBbxqQslfkYi0/Hs/Q4PBzln8e8mqN/Nhyb1geXuGzZDUa+u60aWMaUilp4S9us3x+QXkixLqKc/asVCzo0SNaxhgNsPCKcNaAMSrevn27dfZnjo8/Np35OGqU9HzmTIox87uFfv1o0I6P/Lu6Ulw6MZE8foDycvfvt/z4a9cW/3llZVHaIxcWFxepLx8XTrmXyJ8D0msuKjw80r07eX7ca+MCKr+d50IoDy3wZXwgTu6ZygcHC+6HXyTkolpQpPlfLpBBQdJyR0e6k7pzx1To5dtxVCoqUlWrFom1PKQh36ZgLLjgsoJiLRdd3heyVSvpTkVulzx2XtowhjzUUxEIsa6C5OYCe/dKGRp370p1NnhRfe7VlRbG6FhLlhSfjlYSvLyoYP3vv1McUi7UBWnSpPCyTz4xv26PHlI+b/fuUoU1c0yeTAOulnhbN2+S2PLYp6cn3QXwjjLcQ1SrSTyOHjUdEOPv80HA8HDJGy8oeDy3nd8Z3bolxa258PI4NRcneRhB7sHK91vwoiB/T/6cX+jMTXKqX9+0qp9cTPk+5DFx+fnLwxny7eX2OzlRSE/ebk2+b3mN8SlTKCQVECDdYfAwSEHvvWBoRylc8CsqK0SIdSXlzh2KRXt6Uoy0TRvK0GjWjGZl8QFFtZrCH4yRp2fNOlTR0cDcucq369OHskp4i6Xu3ckTrl2b0s5SUmiwqmVLErrXX7eezQCFMDgtWlA4IyGBpi0DwLRpxWfEFCQpiQa0VCq6wPBZm46OkqfMGE0uuXFDahdWty5tB1CmAk//atPGtFdj/foUypLHlDt2lGqON2tG/+87d+j/7edHMdOTJ00nbsg9a7l3aWMjhcYKDhw+9xydn3ymX5cudEHp08f858GzU/hx5RkW/K88/5sjF0/+vFYtaZIWP/du3eh/VjB+Lr/D4Oep1VIXIh7m4MWYCl7ACm5bEqzhnZcGxWJdp04dHDlyBJ4F3KCUlBS0bNkS169ft5pxTyObN0s1dnlBfsaoGL3BQKlIBgN5uenp0ig6Y/SjLy16PQ2CWcJbb9HEEjkffkg/Nr2e7KlZ0zS+7eREj/KuTODrWzgGbSnXrklpbS4uppkWPLOmY0d6Lv8f8LoXPM7Jf+gFm+o2bEizFXkanBwuNi1bSt5sw4bS4FpBMfLyogFmuXfL+zPy9dq0oVmcDg50l+DvL5UDdXSUjl2cMHl4mLY9k19oANpfUpLpufIYv7291IyhRQu6GPHaHPL9yc+tYBiEh3PkIR8+0Cn/rAveUVjDs64oFIv1jRs3zDbFzcnJwZ07d6xi1NMKH/Hn3bN5NoFWK6VnOTpKMxEzM6UR/5CQkn+RGAP+/W/LJqZwIiLIW372Wfrhd+9eeGIDrydRVYmLkwZP+QCtfICQC0qNGkWHcCwVB3MhCScn09BT9+5SH8bkZPPi5uZGdy1ysfPzkybiAKZhAo7SO7KGDaXsH5WKLkznz0v79vUlIZaLNfemvbwoDCQP23TsSA0s5BcLlYqyUNLSpFRJeccgLtw8J5uLtbkYesFQi1Jq16YLasGBzfLE4sNukrklO3bsgKurq/G1Xq9HdHQ0gvhIhEAxt2/TbS1Ps0tPl+pD63RSg1Y+wJaeLon7M8+ULtPj+HFlQg1IE1xcXIDnny/5seWkpUl2tGlD537pkpSNUJ7wMrJqNQm0hwfd/nMx4PHU3r2t6221bWsqBnJxsbeXxM/fn/Lo+axVvh4vVCT3rAMCSKzd3OhcatSwnr1165ItLi4k1ioVXbj8/QuPmXAb69Sh0J4cHhuXx8i5sAcF0R2LnR2FmeQDtPKLjnzKurkQSGnEOjVV2m9Fzf+zWKx5R3GVSoXBgwebvGdra4ugoCDF3c0FRF6e1FYrO5v+8inivDQpv5nhrbf0esr4SEwsXXfyc+eAqVMtW/df/6Iu12XxZT1yhP7yHyJ/rVJJ1dzKi+xs8lz59Gt3d9M6GXJxVnIRKU4o/PxoIFGev9usmVRFsSAaDb3v5kYCxtt9eXgUzqzQaIrv/FMaVKrCXnmHDkWvK/emzb1f8DX/fHktD3nqnnxyDVA4LbDgICbfJ6/2pwRvb7rL4nn1FYHFYm147EoEBwfjyJEjYvaiFTAYKGc1K4vENzubvCWtlq7kBgMNhN25Q2lr8gkNAH1xStP28vhxGk23hMmTrVt3OiuLhCgzUyq+5OhoOpmE5wQbDFT/onnz4mdfWovTp+kzdnYm8XNyktK2GKOBty1blFf9K06sW7cuvMzD48kX4tq16bFxY2Hv0deX8sKV2mIpBffh7198oSWVivLNzW0rz9rgf3mLuYKYG3iUZ+LI7ywKHkOnk1IRlcAHbfmkpIpAcfQlrqhLvUAx+/ZJ9Tr4bXdmptQ4lnuTwcH0BeNfvvv36Za8NPBpu5ZiLnWuJNy/Tz+mq1el87l5k35EXl704+Q1s3Nz6ceRl0fPT56kH01QEIWBeN0Ra8EYefR8UomXF3lRPO2rfXtpYExeIa8yYO62X54VUhBbW+sMSMuxpGFUwYE/uT3NmknOh0ZT9OSTgvF9uZctP4a54/bo8WQbizpmacIo1qBEofLo6GhER0fj3r17Ro+b87+C6QECI+fOkffBeyNmZFBcNiODvgS8w4teX/hWTf4lKW3M8dgxy0MfHGsMqpw9K3WU4fFFrZZ+lB4eFG7gXhHvRMPvOGxtpe42V67Qjy81tWS3tEXBhdrVVarhbGtLVfECA009KqW1lYGy/aGbG2xs0oTix+by7Xv1Kt3xPD0pRZH353wSvEEyH1vhg5By5BffDh3MF9qSZ3UUTAOUh0cKftbydUpLRWWEKP4JTp06FdOmTUPr1q1Rs2ZNqCryUlOFuH+fROb6dUpt27+fRIjnHatUUnpV69bW/0JcvQqMHUs/UqWzEPv2Ne9FpqWRYKank7drb08/MHd3uvDo9dIP8sgROidPTynVkNc2zs4mgeSeIG+7JG8lxvPKAam5aW4uDbBlZVGVP2v0ynN2Jhvt7elC0rWrdabpl64t9ZMJDzfNguBZEgEBZXM8lcq0fsyT6NKF/o9OTlJD5eJivzzFs6hj80eXLtIMUB5mOXOmcHU9azgbFS11ik9h0aJFWLZsGd58882ysKfakppKDycnKn6k15PIpaRIMxEdHclbsbZQnz4NfPopPVcq1KNHU8pYQe7fp5gzT5v6+2/6Mqek0HL+xZZnK7i5kajm5UnCYmMjxapVKrpN3bVLeo+nZPEYNm+EYGsrlWNVq+lC2LSp8nhiRgZdYHiYycVFqu0BWK+einxCSFkgF77nny+Z51+WqFSS+Do4UEipuBmrRdG6Ne1r3z76Ky8fyzM1+Ofcvj2llapUUq2U0p5DRaLY/NzcXHQoarhXUCR375IwpKWRt/PoEYlEZia9r1LRQJG1B9Di4iShVkJUFF04eB53SgoJFy+nGRcntfNKS5OyEfhAHE9xS0+XvGUPD8ljlVdH415ncDC937cvCfr27aY/ND6QxKvOZWWRV87LkZ4+TYOgT4olX7pEttSoQXcceXlkp1ZLFw6Nhm7vrTlxR56xUNZYqx5MWVLSUB6PactDGgVjyfw7JT+GRkNzEUqKvX0VFOu3334bK1euxKTyLINWDUhNJbHOzibRS0mh17yFEZ+NaC2SkmhAriTTxQFp1B6gHHB5810+e8zXV2q/xG13c5Mm7/DYs7MzCQgvgM+9zMaNyRPKyaFbV3lKoK0tFYRPTaULwvbttE/uafN9cO8sP5+87GPHKMZc1ABsWpr0qFGD7NNo6OLJS5sCpfthC8qHguKpVkupj+beK81YT0n6fVobxWKdnZ2NxYsXY9euXWjatClsCyjM119/bTXj9Ho9pkyZgp9//hkJCQnw8/PDkCFD8Omnnxpj5YwxTJ48GT/88ANSUlLQsWNHLFy4EPXq1bOaHdYiJ4dE5c4dEjgbG5rlxfOqS+oRJSVRzz2Amqra2UmvS8ILL5i+Tkw07X9nMEhpbba2dC55eZJnmp8vFbPPzKSsCu5th4dL3rE83OPmVlhgbWwkMe7bl47Li8136kQFpnicOiuLjp2ZSXWnGzQwHxO9dUvKHOCDnfb20sxAtdo6sW9zVLRnVl0xl79d0NNWq01nR5aEtm0rNrykWKxPnz6N5o+r4Jw9e9bkPWsPNn7xxRdYuHAhli9fjsaNG+Po0aMYOnQoXF1d8cEHHwAAZs+ejXnz5mH58uUIDg7GpEmT0LNnT5w/fx52FZUQaQbGSMRatybvjxfoAUzjeZbw998UpnjzTepDyOPCAHXB7t3b/HYFu30URf/+0nOec+ziIpXV1OtJRHmtBx464FXWeNlOQOpkolJRqlhRg0o8Q6Y41GrTyR3yqcWA1DU7L49mQvJ0MHmp2KwsqX8gPzd+kXByov0pKfIkqBjMZb9Yun5J8fEp/T5Kg2Kx3rNnT1nYYZZDhw4hIiICfR6X/woKCsIvv/yCw4cPAyCves6cOfj0008REREBAPjxxx/h4+ODDRs2YODAgeVm65PIyZHErk2bku0jLc20l+DMmebX27bN/PL58+lL+9JL0rLwcCq+tG8f8M03tMzDQ5rSztPm+O2lwSB51mFhNIAj74/HQxOtWwM7dkgdpXlTWGvSty/FnC9coAuHmxsd6/59ujDm51OZUkAKjfBBKK2WvHA7OzpfvrxVK2UXTiXI4/OC0qFEfKvLHU2lLpHaoUMHLF68GJcvX0b9+vVx6tQpHDhwwBhqiYuLQ0JCAnrIMt1dXV3Rrl07xMTEFCnWOTk5yOEpGADSeEfYMsJgkOoeK+HmTeD99+l5795Fi7AldOtm/ks7bpz0vsFAeeBnz0rV37jX7+gozSjk2Rw1alDmwR9/0C0iz0/m8IJH9epZfwIGp25dys/dtYts4xcaQAo7qdU0qMhtkuf6uriQcPMGtPLyqoKqScGLYmWavFQaSiTWR48exZo1a3Dr1i3kFqjpuG7dOqsYBgATJkxAWloaGjZsCI1GA71ejxkzZiAyMhIAkJCQAADwKXB/4uPjY3zPHFFRUZiqdFZIKXj0iAROSVQmK0sSaqB0Qv2//1HcuCCvvmr6mqfoHT5sOivMycn0ta2tNFij0xVdd6J7dxo4bNCg5LZbglZLF7ObNym04eVFFxZHR6rxwTtn80wWnt7l4CB54+VRnKdgvQpBySmY3qpSUQVIgNI/5bMfK3rmobVQnNG7atUqdOjQARcuXMD69euRl5eHc+fOYffu3SaV+KzBmjVrsGLFCqxcuRLHjx/H8uXL8eWXX2L58uWl2u/EiRORmppqfNy2JJBbAnhnD97JxdKPZ+VKij1bgxUrCgv1Z5+RkP7jH0Vvx6dc827ZAP1AatUC2rWjHNYnYWtLU5DL64dSuzZVdNNqKRTj4yOVL/X0JPH28iJhtren97hw29hYNl26NLRoAXTuXLbHeFow953iA4h2dqahrIrs7mJNFHvWM2fOxDfffIORI0fC2dkZc+fORXBwMN555x3ULE1VITOMHz8eEyZMMIYzQkNDcfPmTURFRWHw4MHwfZxPk5iYaHLsxMRE4yCoOXQ6HXTlkIy6e7dUCJ5nSDyJuDjK6LAW5iZ1tG5tvnAQIHXZ4INwAD13dJQ8l8pM/fpSzjdAgqxSSV41r/UBSNPJu3cvH8+6vBsuVGcKVha0dN2qjOLTuHbtmnHAT6vVIjMzEyqVCh9++CEWL15sVeOysrKgLvBJazQakwqAvr6+iI6ONr6flpaG2NhYhIWFWdWWkpCVRXnCjJFn/aSUsF27TLuqFMeIEVK82VokJlK2iI0NeSZ8OriNjWnpzsqMrS0Vsm/bVppQw/sm8intvOqaVkshmoqqTywoHZZ6y9UlDKLYs3Z3d0f64/YV/v7+OHv2LEJDQ5GSkoIsPunfSvTt2xczZsxAYGAgGjdujBMnTuDrr7/GW2+9BYBSBceMGYPPP/8c9erVM6bu+fn5GetvVySMUa3hkBCKWz9puuu8eYWXzZ8PTJxImRnffUeZDgaDVK60TRvrhUxu3qS/jo6FO3xUVA3fkuLrC7z4IpUyrVuXBnh5B5XQUBLse/esn6EiKB+UZoM8lWLdpUsX7Ny5E6GhoXj11VcxevRo7N69Gzt37kR3c0UkSsG3336LSZMm4b333sO9e/fg5+eHd955B5999plxnY8//hiZmZkYMWIEUlJS0KlTJ2zfvr3Cc6wNBspKyMujlDveAaYo5IOJnLlzKdNhxQppWcFbaQcHqRN2QYrr6l0c3BOV18+2coSrXFCpSLBVKqmhKi8+5OxsvbofgoqhYKnUouAFwqo6isV6/vz5yH489euTTz6Bra0tDh06hP79++PTkhShKAZnZ2fMmTMHc+bMKXIdlUqFadOmYdq0aVY9dmm5eJGmkzNGnbWLqs0LUI0Q7tVyVq60PN93+HBg1qzCy5WM9yYkSClPdnbSjEU+OFPZCgNZinxmW9OmFWuLwHqoVJY3EaguMWvFYu0ha12hVqsxYcIEqxpUXbh2jQYXvb3pdttcOcnoaAqNFOyG5uWlbGJGUQNXSga0bt2SWmrxwklcrP39Ld+PQFAeKGlT9tSKtUajQXx8PLwLjDglJSXB29vbbOfzp5W8PAofmPMAbt8uusjSwoXKjhMYCHz8MYl8ZiY1Fig4QcUSeAstXhWPNyWt6Gm2AkFxPCnEUV1y2xWLNStivmxOTg605dEgrwrBO8CY4+RJ88sHDy5ZQadOnaTnc+cqqxKWlCSFOnhVO0CyXXjWgqrMU+dZz3ucqqBSqbBkyRI4ye7T9Xo99u/fj4ZKWkdUYzIzaapzTo75hp8A8MMPhZf99pt1YsPBwcrWv3ZNEuvcXFOxFtdfQVWnqo63FMRisf7mcZUfxhgWLVoEjezeQqvVIigoCIsWLbK+hVWQ+/cpv1op1mg9VBTZ2dJU7Dp1pOU5OdJUcq1WKuYfEEAevlLhFwgqguLu/p66MAjvat6tWzesW7cO7mVV9LcKExMjNfnMylI2kWT0aGXpRXxowNIv4unTJMgpKab96U6dkup98Op5arUoEyqoOvTuXXSoo25dKuJlSXmEyo7iaM6ePXtMhFqv1+PkyZN4+PChVQ2rijx4QJ5pbi41jJUlzpjw5ZfS865dgfXrzfc5NEdeHsXCjx0Djh+3rOQmr7Xl5kaDjryVGId3GXdwkNpnCQRVheL6W1aXCTFACcR6zJgx+O9//wuAhLpLly5o2bIlAgICsHfvXmvbV2XIz5cauPLCTeZm/eXmUmdzztixxXvHfKo63/bECarfrFbTdsnJ5ifEyDlzhkTYx4cE+cIFqqx3/z69b2tLy3n3lury5RYIHB2lEF9VR3GUdO3atXjjcQX8zZs348aNG7h48SJ++uknfPLJJzh48KDVjawKZGRQiIH3KixqGrO8l6ElXL1qGrrQaChcwUMWfHAwJYU8eXl0ijHa3mCgWLW9vVTrQ6+nolFqtbQ/3uy2uoyeCwQBAeQ0WbkgaIWg+GeZlJRkrHa3bds2vPrqq6hfvz7eeustnDlzxuoGVhXu3SPPOiCAZsrx3oEFmTFD2X4fPpSazzo6SqU/fXwoJs5DG8nJwJUrptsePUoibmMjVZvjpUH5CLlGIxVs0miqz2CMQMCpDkINlECsfXx8cP78eej1emzfvh3PPfccAKqQp3kKf+kHDwKbN5NQ5+WRB1tcnvSdO9LzJUuK3/fhw7Qvb2/6W6sWPecTXrRa6mzCC+irVDTNHaDp49yTdnKiMEfNmlLBfd7iSh4CAYRYCwSVFcVhkKFDh2LAgAGoWbMmVCqVsaVWbGzsU5lnnZxMQn3jhtTuylKKyhZ53GISajVNbvHwoLAFT22Xty3iE1nS06myH//74IEkwjodCXuLFvT66lUqYpSUJBVt4mmDZZk+KBAISo7in+aUKVPQpEkT3L59G6+++qqxiL9Go3lq64TwyS88Z7k0MCaNbtvYkFDz3oJqNaUg3blDlfxcXekCcf06CTMfiOTRKC8vqRmspyftt0kTilXb2tL+dTr6y+PU1WUCgUBQ3SiRH/UPM/2gBg8eXGpjqiIGA4mkm1vRsxU5vGkrALz8svl1jhyRCuU7OZF48ma5rVpRvJn3P+TcuEEeMo875+aS6NvaSqPg8nZSffoAa9bQ+3Z2tE337lRYSoRBBILKSYnEOjo6GtHR0bh3756xawvnf//7n1UMqyro9STYPHWvOMaPl54/TqgpBG+p5esr9QXUaIqvMsanhfOJLTY2UkimR4/CFxGe8SHPKuHrCLEWCConisV66tSpmDZtGlq3bm2MWz/N5OeTMKamKqtyZy7cEB9Py11cSHxVKsuKKHGxtrEhsbWxIbvkImxuGzs7esgnDjzl/06BoNKiWKwXLVqEZcuW4c033ywLe6ocjNGgno8PxZGLW+9J/P03CbWnp9TlxBJ4fJs/bG1pJmVxFw+VSgqbyBE51gJB5UTxTzM3NxcdOnQoC1uqJAYD1QHJzJTqdZjjcXOdIrl/n4TS3Z285K5dLfdy+WxGuVg/qbsL96xtbCgmLt+XQCCofCj+ab799ttYuXJlWdhSJbG1pZADn7pdFI/rYAEApkwx/769PQmo0v6AbdpI3jWv86FSUQpfUfDYOJ8VyRExa4GgcqI4DJKdnY3Fixdj165daNq0KWwLuG9ff/211YyrCjBGj7S04rNB5FmNLVuavvf33yT63KtWGjfmYRM+A5EXY3qSZ21u4FKjoUplAoGgcqFYrE+fPo3mj+tnnj171uS9p3Gwkec25+UBzZqZX6eoePXDhzRBhTESXBcXEtiQEOV2cHGWi3XBFD9L0Giqz/RcgaA6oVis9+zZUxZ2VFkMBqB27eL7FMpLksq93StXyJPW66UQiEajrA423yf3lLloq1TFd1Q3d10NDaUJNU/KFxcIBOVPqSYX//333wCAWkraaFcjGJOayhaHPP966VLT95ydySvnmRlKujZz+PF5xxfeQKC4bjXmxDooiLxq0VdCIKh8KB5gNBgMmDZtGlxdXVG7dm3Url0bbm5umD59eqEJMtUdPiHmSWJ9+rT0vGCNa09PEkdHRyA8vOS2yEub8lBIcXYVFbESQi0QVE4Ue9affPIJ/vvf/2LWrFno2LEjAODAgQOYMmUKsrOzMUNpDdAqjMFg2g28KA4dKrxMr6dwBQ9/aLXmmxVYCp/Ywr3rJ4UynsLhBYGgSqNYrJcvX44lS5bgpZdeMi5r2rQp/P398d577z1VYm3JRBcAiI0tvOzBAwo58NodRTUrUIJcrGXN54tE3jhXIBBUbhSLdXJystlSqA0bNkRycrJVjKpKWCLYjRoB588D8o9Nr5dqVYeElD4DIyAAuH2bnqvVT/b2SxIbFwgEFYfimHWzZs0wf/78Qsvnz5+PZkXlrpWCO3fu4I033oCnpyfs7e0RGhqKo0ePGt9njOGzzz5DzZo1YW9vjx49euBKwZYpZQQX6ieFFM6fp7/yMEdqKoVBmjSxfqocnyDj4GDd/QoEgopDsWc9e/Zs9OnTB7t27UJYWBgAICYmBrdv38a2bdusatzDhw/RsWNHdOvWDb///jtq1KiBK1eumHRXnz17NubNm4fly5cjODgYkyZNQs+ePXH+/HnYKamsVAL4hBgl08I56enU5aW4rjJKKLif4jo+CwSCqodisQ4PD8elS5fw3Xff4eLjHlKvvPIK3nvvPfj5+VnVuC+++AIBAQFYKst3Cw4ONj5njGHOnDn49NNPERERAQD48ccf4ePjgw0bNmDgwIFWtaek1KlDDQIed0BDRgaJqYOD9Yr9N2hAjQZOnKBBTzFtXCCoXpQoz9rf379cBhI3bdqEnj174tVXX8W+ffuMg5jDhw8HAMTFxSEhIcHYWgwAXF1d0a5dO8TExBQp1jk5OciRJT+nFVcurxgs9ayvX6e/fNAvKUmaAGMt1GqamCMfZBQIBNUHxT/ppUuXYu3atYWWr127FsuXL7eKUZzr169j4cKFqFevHnbs2IF//etf+OCDD4zHSUhIAEBNfOX4+PgY3zNHVFQUXF1djY+AgIAS2WfJ4KI89ZwP+iUmSk1urQ0vziRCIAJB9UKxXERFRcHLy6vQcm9vb8ycOdMqRnEMBgNatmyJmTNnokWLFhgxYgSGDx+ORYsWlWq/EydORGpqqvFxm6dRKMQSz/rePem5PErk5FS4oJM14BknQqwFVYGuXbtizJgxFW1GlUCxWN+6dcskbsypXbs2bt26ZRWjODVr1kSjRo1MloWEhBiP4+vrCwBITEw0WScxMdH4njl0Oh1cXFxMHiXBErGW32w4OkrPbW1Nxdta8NNWqURBJkHlYMiQIVCpVIUeV69eLZPjmbsAJCUloVevXvDz84NOp0NAQABGjRplEgJdt24dnnvuOdSoUQMuLi4ICwvDjh07ysTGkqBYrL29vXFaPn/6MadOnYKnp6dVjOJ07NgRl+RdZgFcvnwZtWvXBkCDjb6+voiOjja+n5aWhtjYWGOmSllSXBjkr7+Al14CDh4s/J5aLRVbKkssmRgjEJQHvXr1Qnx8vMnDnNNXVqjVakRERGDTpk24fPkyli1bhl27duHdd981rrN//34899xz2LZtG44dO4Zu3bqhb9++OHHiRLnZWSxMIR9//DGrXbs22717N8vPz2f5+fksOjqa1a5dm40bN07p7orl8OHDzMbGhs2YMYNduXKFrVixgjk4OLCff/7ZuM6sWbOYm5sb27hxIzt9+jSLiIhgwcHB7NGjRxYfJzU1lQFgqampiuxLTWVs4ULGVq5kbNMm04fkd0sP/t7UqfS3rNi0ibGNG8k+gaCiGTx4MIuIiDD7Xnh4OBs9erTxdXJyMnvzzTeZm5sbs7e3Z7169WKXL182vv/gwQM2cOBA5ufnx+zt7VmTJk3YypUrTY4FwOQRFxdn9thz585ltWrVKtb2Ro0asalTp1p8rgUpqbaYQ3E2yPTp03Hjxg10794dNo9HzAwGA/75z39aPWbdpk0brF+/HhMnTsS0adMQHByMOXPmIDIy0rjOxx9/jMzMTIwYMQIpKSno1KkTtm/fXuY51oDlk2IKYq10veL2n5dXulojgioAY9RTriJwcCiTW8MhQ4bgypUr2LRpE1xcXPDvf/8bvXv3xvnz52Fra4vs7Gy0atUK//73v+Hi4oKtW7fizTffxDPPPIO2bdti7ty5uHz5Mpo0aYJp06YBAGqYKex+9+5drFu3DuHFVE8zGAxIT0+Hh4eH1c+zJCgWa61Wi9WrV+Pzzz/HyZMnjbMKeWjC2rz44ot4sZjOsSqVCtOmTTP+Y8qTomLW5upIc/MMBsrYEEIqKDVZWRUX68rIMB2EeQJbtmyBk8zWF154oVBWGRfpgwcPGvu8rlixAgEBAdiwYQNeffVV+Pv746OPPjJu8/7772PHjh1Ys2YN2rZtC1dXV2i1Wjg4OJgdtxo0aBA2btyIR48eoW/fvliyZEmRNn/55ZfIyMjAgAEDLD7PsqTE9azr1auHevXqWdOWKofBUDhunZZmOqjI4TPxGaOYdVm2zmrUCLh8uez2LxAopVu3bli4cKHxtaMZob9w4QJsbGzQrl074zJPT080aNAAFy5cAADo9XrMnDkTa9aswZ07d5Cbm4ucnBw4WFhb4ZtvvsHkyZNx+fJlTJw4EWPHjsV3331XaL2VK1di6tSp2LhxI7yVdgMpI0rVfOBpp6BnnZwMDBliuk5kJNC6tbTOo0f03NKKfSUhMJAegmqOgwN5uBV1bAU4OjqirhU8lP/85z+YO3cu5syZg9DQUDg6OmLMmDHIzc21aHtfX1/4+vqiYcOG8PDwQOfOnTFp0iTUrFnTuM6qVavw9ttvY+3atSYT7ioaIdaloGDMuqBQA8Brr5m+Pn8e8PenWYxPaYMdgbVQqRSFIio7ISEhyM/PR2xsrDEMkpSUhEuXLhlTeA8ePIiIiAi88cYbACiufPnyZZMUX61WC71e/8Tj8WYp8tnMv/zyC9566y2sWrUKffr0sdq5WQMh1qVA7lnHx1u+nasrlTQVCAQS9erVQ0REBIYPH47vv/8ezs7OmDBhAvz9/Y21f+rVq4dff/0Vhw4dgru7O77++mskJiaaiHVQUBBiY2Nx48YNODk5wcPDA9u3b0diYiLatGkDJycnnDt3DuPHj0fHjh0RFBQEgEIfgwcPxty5c9GuXTvjLGh7e3u4VoJJCyWaFMPM3MMzxqw+KaayIxfrd94p/P4//mF+Ozs7MWFFIDDH0qVL0apVK7z44osICwsDYwzbtm2D7eMUqk8//RQtW7ZEz5490bVrV/j6+qJfv34m+/joo4+g0WjQqFEj1KhRA7du3YK9vT1++OEHdOrUCSEhIfjwww/x0ksvYcuWLcbtFi9ejPz8fIwcORI1a9Y0PkaPHl2eH0GRqJg55S0GjUaD+Pj4QkH3pKQkeHt7W3T7UdlIS0uDq6srUlNTFc1mTEwEVq+mcEb//tLyn3+mUKK5GYpXrgDt2wOP7/IEAkE1pqTaYg7FYRDGGFRm8iszMjLKJbe5MsGzQfLypGWvv05peUX9X3JyntzFRSAQCApisWyMHTsWAOU1T5o0ySRVRq/XIzY2Fs2bN7e6gZUZLta8BCpQeECxIHl55JELBAKBEiwWaz4/njGGM2fOQMs7vYJGX5s1a2aSrP60wBiwbJn0+kmTusp69qJAIKieWCzWe/bsAQAMHToUc+fOLXX8pTrAo/0tW1o+CUWtFr0RBQKBchRHT+Uttp52kpNJsHnHF962qygyMsizrqhyDgKBoOpSoqGuo0ePYs2aNbh161ahmUPr1q2zimFVgZs3Ab2eMkIAel4c+fnkVYsca4FAoBTFedarVq1Chw4dcOHCBaxfvx55eXk4d+4cdu/eXSkSx8sbg0Eq3LR7d/Hr5uSQF16WdUEEAkH1RLFYz5w5E9988w02b94MrVaLuXPn4uLFixgwYAACn8KCFI8eSc8fT7IqksxMEmvRcksgEChFsVhfu3bNOGdeq9UiMzMTKpUKH374IRYvXmx1AyszGRnA1KnS6zZtil//wQPKsX7K0tEFAoEVUCzW7u7uSE9PBwD4+/vj7NmzAICUlBRkPWUjZytWmL5u3PjJ22i10oCkQPC0IxrmWo5ise7SpQt27twJAHj11VcxevRoDB8+HIMGDUL37t2tbmBlZutW09dPEmGVisRaIHiaqAwNcwHggw8+QKtWraDT6YqcwMcYw5dffon69etDp9PB398fM2bMKBM7laI4G2T+/PnIzs4GAHzyySewtbXFoUOH0L9/f3z66adWN7A6odGISTGCp5NevXoVSvs1126rrHnrrbcQGxtrtuk3AIwePRp//PEHvvzyS4SGhiI5ORnJycnlbKV5FIu1vB+ZWq3GhAkTrGpQVcXf/8nriPCH4GlFp9OZbbNVkIcPH2L06NHYvHkzcnJyEB4ejnnz5hm7UiUlJWHUqFHYv38/Hj58iGeeeQb/93//h0GDBgEgL37fvn3Yt28f5s6dCwCIi4tDUFAQ5s2bBwC4f/++WbG+cOECFi5ciLNnz6JBgwYAUK4d2J+E4jCIQOJxGVw0a2Y60GiOtDTA2bnMTRI8RTBGGUYV8SirTkdDhgzB0aNHsWnTJsTExIAxht69eyPvcbU03jB369atOHv2LEaMGIE333wThw8fBgDMnTsXYWFhGD58OOLj4xEfH48ACyc2bN68GXXq1MGWLVsQHByMoKAgvP3221XXsxZI8EkwgwYBT2rTxhg19XhSxohAYClVqF9upWmYWxzXr1/HzZs3sXbtWvz444/Q6/X48MMP8Y9//AO7nzSJohwQYl0KuFirLbg/4Z7IUzhvSCCoNA1zi8NgMCAnJwc//vgj6tevDwD473//i1atWuHSpUvG0EhFIcS6FDxu4WZRLPryZeCZZywTdoHAEqpQv9xK0zC3OGrWrAkbGxujUAPUFxKgDlkVLdaKpYNX3zPHggULSmVMVYNPM7d04NDBAdDpys4ewdMF75dbEY+ymIUrb5jLKa5hbrNmzVCnTh1cLlDy0tKGuQXp2LEj8vPzce3aNeMyvu/atWuX5JSsimKxfuWVV3Ds2LFCy+fOnYuJEydaxaiqAv8+WCLWKpWYuSgQFIe8Ye6BAwdw6tQpvPHGG4Ua5u7cuROHDh3ChQsX8M477yCxQDcPecPcBw8eGLuYX716FSdPnkRCQgIePXqEkydP4uTJk0avvEePHmjZsiXeeustnDhxAseOHcM777yD5557zsTbrigUi/V//vMfvPDCC7h48aJx2VdffYXPPvsMWwvOEqnmKBFrV1fq1SgQCIqmrBrmAsDbb7+NFi1a4Pvvv8fly5fRokULtGjRAnfv3gVAqcibN2+Gl5cXunTpgj59+iAkJASrVq0q18+gKBQ3zAWA2bNnY968eThw4ABWr16NmTNnYtu2bejYsWNZ2Ghk1qxZmDhxIkaPHo05c+YAoFSecePGYdWqVcjJyUHPnj3x3XffwcfHx+L9lrSppZMTpTEtWmS+Oa6c06ep3nXbthbvXiAQVHEqtGEuAHz88cdISkpC69atodfrsWPHDrRv375UhjyJI0eO4Pvvv0fTpk1Nln/44YfYunUr1q5dC1dXV4waNQqvvPIKDh48WKb2AMqyQQwG0XtRIBCUHIvEms/8kePv7w8HBwd06dIFhw8fNialf/DBB9a1ENQ5PTIyEj/88AM+//xz4/LU1FT897//xcqVK/Hss88CoNuokJAQ/PXXX2V+AVGSDWJjA7i5lak5AoGgGmORWH/zzTdml2s0Ghw8eNDoxapUqjIR65EjR6JPnz7o0aOHiVgfO3YMeXl56NGjh3FZw4YNERgYiJiYmDIXayUxaxsbKXtEIBAIlGKRWMfFxZW1HUWyatUqHD9+HEeOHCn0XkJCArRaLdwKuKw+Pj5ISEgocp85OTnIyckxvk5LS1NsF2OWi7XBQOs8hb0ZBAKBlajUUzRu376N0aNHY8WKFbCzYt5bVFQUXF1djQ9LawfIkadxPkmsHz6kuHYJrgkCgUAAoARi3b9/f3zxxReFls+ePRuvvvqqVYziHDt2DPfu3UPLli1hY2MDGxsb7Nu3D/PmzYONjQ18fHyQm5uLlJQUk+0SExOLrQswceJEpKamGh+3b99WbJs8pPGkAcZr1wB7e+DvvxUfRiAQCACUQKz379+P3r17F1r+wgsvYP/+/VYxitO9e3ecOXPGmLx+8uRJtG7dGpGRkcbntra2iI6ONm5z6dIl3Lp1C2FhYUXuV6fTwcXFxeShFLlYF+VZZ2VJg5COjkDnzooPIxAIBABKkLqXkZEBrZl2J7a2tiWK/RaHs7MzmjRpYrLM0dERnp6exuXDhg3D2LFj4eHhARcXF7z//vsICwsr88HFxxUbARQWa4OBvO1z56TKZDpdxVVIEwgEVR/FnnVoaChWr15daPmqVauM8/fLk2+++QYvvvgi+vfvjy5dusDX1xfr1q0r8+MW5VlnZgLHjgGHD1NXmIwMqUOMjSibJRAISohi+Zg0aRJeeeUVXLt2zZjbHB0djV9++aVQfdqyYO/evSav7ezssGDBgnIvIsU9a7XatKjNuXOUT52SQt50fj7g4iK6xAgEgtKhWKz79u2LDRs2YObMmfj1119hb2+Ppk2bYteuXQgPDy8LGysl3LMuOLio0VAjgsxMGlTU6wEvr/K3TyAQVC9KdGPep08f9OnTx9q2VCm4Zy33mHk3GAcH8q5tbMirtkJddIFA8JQjoqglxJxnnZxMwqzV0l+9nmLVtrZA48YVY6dAIKgeKB5gVKvV0Gg0RT6eFsw1HkhIIJEOD6fa1RoNNclVq4E6dSrGToFAUD1Q7FmvX7/e5HVeXh5OnDiB5cuXY+qTWnxXI+QDjJysLPKqXVxocJExCoV061YxNgoEguqDYrHmHRvk/OMf/0Djxo2xevVqDBs2zCqGVXYKetZ5eeRFP66RbuwKY2sr8qsFAkHpsVptkPbt25vMJKzuFPSsb98mj1ou0jod8NJLFWOfQCCoXlhlgPHRo0eYN28e/P39rbG7KkHBAcYHD4D69YEuXeh1z57SVHOBQCAoLYrF2t3dHSrZLBDGGNLT0+Hg4ICff/7ZqsZVZsyl7jk7Sw0GVCoxEUYgEFgPxWL9zTffmIi1Wq1GjRo10K5dO7i7u1vVuMpMwZi1RiPEWSAQlB2KxfrZZ59FQECAiWBzbt26hcCnpML+w4f0NzmZ/up0FWeLQCCo/igeYAwODsb9+/cLLU9KSkJwcLBVjKoKfPwx/X30iJoK2NtXrD0CgaB6o1isGWNml2dkZFi1m0tlR36qvA5IgWquAoFAYDUsDoOMHTsWADXF/eyzz+AgK3ih1+sRGxuL5s2bW93AykpkJDB5MvVV5NX1atasaKsEAkF1xWKxPnHiBADyrM+cOWPSgECr1aJZs2b46KOPrG9hJYXfYLi6AtnZJNh8QoxAIBBYG4vFes+ePQCAoUOHYu7cuSVqhVUdUauBnByqCSKyQQQCQVmhOGa9dOlSE6FOS0vDhg0bcPHiRasaVtnhE17UauD+fZENIhAIyhbFYj1gwADMnz8fAM1cbN26NQYMGIDQ0FD89ttvVjewssLzrHkGo2jZJRAIypISdTfv/LhN9/r168EYQ0pKCubNm4fPP//c6gZWVvR6+qtWU6Gmgh1jBAKBwJoolpjU1FR4eHgAALZv347+/fvDwcEBffr0wZUrV6xuYGVFLtZubiJeLRAIyhbFYh0QEICYmBhkZmZi+/bteP755wEADx8+fKryrHkYxMaGalg//hgEAoGgTFAcaR0zZgwiIyPh5OSE2rVro2vXrgAoPBIaGmpt+yot3LPWaEisRdqeQCAoSxSL9XvvvYd27drh1q1beO6556B+HKytU6fOUxWzzs2lvzY2JNgiZi0QCMqSEuUwtGrVCq1atTJZ9rR1O79zh/7yMIhAIBCUJcIfLCHyMIgIgQgEgrJGiHUJkU+KadeuYm0RCATVHyHWJUTefMBMaW+BQCCwKpVarKOiotCmTRs4OzvD29sb/fr1w6VLl0zWyc7OxsiRI+Hp6QknJyf0798fiYmJZW4b96xFfrVAICgPFIv19u3bceDAAePrBQsWoHnz5nj99dfxkLdPsRL79u3DyJEj8ddff2Hnzp3Iy8vD888/j8zMTOM6H374ITZv3oy1a9di3759uHv3Ll555RWr2mEOecw6NbXMDycQCJ52mEKaNGnCtm7dyhhj7PTp00yn07GJEyey9u3bsyFDhijdnSLu3bvHALB9+/YxxhhLSUlhtra2bO3atcZ1Lly4wACwmJgYi/ebmprKALDU1FSLt+nQgTGAsddeYyw52fJzEAgETw8l0ZaiUJy6FxcXh0aNGgEAfvvtN7z44ouYOXMmjh8/jt69e1v1QlKQ1McuLJ/ufuzYMeTl5aFHjx7GdRo2bIjAwEDExMSgffv2ZveTk5ODnJwc4+u0tDTFtsgHGEXMWiAQlDWKwyBarRZZWVkAgF27dhmnm3t4eJRI9CzFYDBgzJgx6NixI5o87p+VkJAArVYLNzc3k3V9fHyQkJBQ5L6ioqLg6upqfAQEBCi2R14bhAu3QCAQlBWKxbpTp04YO3Yspk+fjsOHDxsnw1y+fBm1atWyuoGckSNH4uzZs1i1alWp9zVx4kSkpqYaH7dv31a8D54NYmsrYtYCgaDsUSzW8+fPh42NDX799VcsXLgQ/v7+AIDff/8dvXr1srqBADBq1Chs2bIFe/bsMbkg+Pr6Ijc3FykpKSbrJyYmwtfXt8j96XQ6uLi4mDyUIs8G8fZWvLlAIBAoQnHMOjAwEFu2bCm0/JtvvrGKQXIYY3j//fexfv167N27F8HBwSbvt2rVCra2toiOjkb//v0BAJcuXcKtW7cQFhZmdXvk8DCIra1I3xMIBGWPYrHWaDSIj4+HdwF3MikpCd7e3tBzFbMCI0eOxMqVK7Fx40Y4Ozsb49Curq6wt7eHq6srhg0bhrFjx8LDwwMuLi54//33ERYWVuTgorWQT4oRRZwEAkFZo1isGW/rXYCcnByTjufWYOHChQBgLMPKWbp0KYYMGQKAPHq1Wo3+/fsjJycHPXv2xHfffWdVO8whD4OIbBCBQFDWWCzW8+bNAwCoVCosWbIETk5Oxvf0ej3279+Phg0bWtW4oi4Mcuzs7LBgwQIsWLDAqsd+EvJJMUKsBQJBWWOxWPOYNGMMixYtgkYWqNVqtQgKCsKiRYusb2ElRdQGEQgE5YnFYh0XFwcA6NatG9atWwd3d/cyM6oqIGqDCASC8kRxzHrPnj1lYUeVg6dmixmMAoGgPFAs1nq9HsuWLUN0dDTu3bsHQ4Hpe7t377aacVWB1FQh1gKBoOxRLNajR4/GsmXL0KdPHzRp0gSqp1ypgoOFWAsEgrJHsVivWrUKa9asKfOiTZUdGxsaZLQpURdLgUAgUEaJCjnVrVu3LGypUogBRoFAUJ4oFutx48Zh7ty5FuVAV2f46QuxFggE5YFFN/EFO6/s3r0bv//+Oxo3bgzbAq29161bZz3rKinUdoCei87mAoGgPLBIrF1dXU1ev/zyy2ViTFVBXv5ExKwFAkF5YJHULF26tKztqFIIsRYIBOWNqBdXAuRiLbrECASC8kCxX9iiRQuzudUqlQp2dnaoW7cuhgwZgm7dulnFwMqIXKCtXGhQIBAIzKLYs+7VqxeuX78OR0dHdOvWDd26dYOTkxOuXbuGNm3aID4+Hj169MDGjRvLwt5KgdyzFgOMAoGgPFDsWT948ADjxo3DpEmTTJZ//vnnuHnzJv744w9MnjwZ06dPR0REhNUMrUw87hcMAChBRzCBQCBQjGLPes2aNRg0aFCh5QMHDsSaNWsAAIMGDcKlS5dKb10l5eZN6XkxrR4FAoHAaigWazs7Oxw6dKjQ8kOHDsHOzg4AYDAYjM+rI/L2jqKll0AgKA8Uh0Hef/99vPvuuzh27BjatGkDADhy5AiWLFmC//u//wMA7NixA82bN7eqoZUVUcRJIBCUBypWgnnjK1aswPz5842hjgYNGuD999/H66+/DgB49OiRMTukKpCWlgZXV1ekpqbCxYIgtFygz54FGjcuQ+MEAkGVRam2FEeJxLq6URqxPncOaNSoDI0TCARVFmuKtYi4CgQCQRXAopi1h4cHLl++DC8vL7i7uxfbcCA5OdlqxlUFRMxaIBCUBxaJ9TfffANnZ2cAwJw5c8rSnipBr17A9u1AaKgQa4FAUD5YJNaDBw82+/xppWZN6e+jRxVri0AgeDooUcz62rVr+PTTTzFo0CDcu3cPAPD777/j3LlzVjWussKnm9vYAA8fVqwtAkG1ICMDOH++oq2o1CgW63379iE0NBSxsbFYt24dMjIyAACnTp3C5MmTrW5gZYSLtVpNDXMFAoECcnKAzEzTZXv2AFevPnnb7Oyn9nZWsVhPmDABn3/+OXbu3AmtrOTcs88+i7/++suqxlVW5GItEJQbGRlAampFW6EMgwFIS5Ne6/XAH38A0dHAgwem6+r1Ugumoti5E9i1q/Dy1NTC+6tmKJabM2fOmO0U4+3tjQcV+GEtWLAAQUFBsLOzQ7t27XD48OEyO5ZcrIVgC8qNPXuAffsq2gplHD9ONvMfzfbtQF4ePWJiaBn3svV64MYNadvdu4HNm4H0dNN9GgzAtWumy/bvp/1lZkqCn5wMJCSYxioZo+2Tk598YahkKJYaNzc3xMfHF1p+4sQJ+Pv7W8UopaxevRpjx47F5MmTcfz4cTRr1gw9e/Y0xtOtDf/eaTRWzgbJzDT1QgTW4ebNwj/ukmDuf5ORYb4Dxf37QFKS9PrhQyrXyJj0BcrPp7/Z2UBuLr2XlgbcuUNiYjAAKSkkOIAkNAD9zc+nv+fPS9XF5Pvkx+Prnz1LwidPrzUY6NiZmeSZHjpE26al0b4OHQKuXJHWZ6zwQM3VqxTakNcOBoDr14H4eMljzsykc+vfH/jtN2m9rVuByEjymM+epfXT04G//6bne/fSeidPAv36kYCfPy99Fg8e0HaPHgGbNpHIZ2fTheHwYeDAAcnm2Fg63sGDtJ8ffwSOHaP3UlJom9276fPMzZVs3LSJ1t+8ueJEnilk3LhxrFOnTiw+Pp45OzuzK1eusAMHDrA6deqwKVOmKN2dVWjbti0bOXKk8bVer2d+fn4sKirKou1TU1MZAJaammrR+hER1DL3H/9g7MYNMyvk5prfMDmZsQ0bGMvIoHUMBnowxlh2NmMTJjD2yy/mt83NZSwpibHDhxnLyaFlBgNjej1j+flFG3v9Ou0zI0NaptfT8fh+T52iZXfvMjZzJmMPHjCWmkrHyctjbO5cxg4dYmztWsb+/JPW/eMPxo4cYWzjRrL7hRfo3E6dYmzMGMaOHmUsIYGxwYNp3eRksjcjg7GzZxn79FPads0axmbPZuyf/2Tszh3G0tJ4P2I6Xnw82fPjj7Sv9u0ZmzWLsTNnGJs/n7EmTRhbtIixn3+mz2bgQMYCAhjr0oWxGTOkfckfs2bRcXfvZuzePTrP779nLCiIsc8/p3UmTWJsxw5aLzaWMScn03389BMdH2AsPJyxkSPp+GvXSut4epo/fmgoY82aMebiYrpcrTa/fkkftWox9tJLhZePGaNsP7VrM/bRR4WXN2tmXXurwqNtW/r+W4hSbSkOKN0gJyeHvf3228zGxoapVCpma2vL1Go1e+ONN1h+caJRRuTk5DCNRsPWr19vsvyf//wne+mll8xuk52dzVJTU42P27dvK/pAX6xzjgGMLcFbFf/lEQ/xEI/yfTx8aLE+WVOsFYdBtFotfvjhB1y7dg1btmzBzz//jIsXL+Knn36CRqOxtuP/RB48eAC9Xg8fHx+T5T4+Pkjgt48FiIqKgqurq/EREBCg6Jj56dR9QAP9E9YUCATVjgqapa24ROr169dRp04dBAYGIjAwsCxsKnMmTpyIsWPHGl+npaUpEuzPPtch8s/tqJneCuzPLVAlJ9F0xuxswMMDiIsDXnqJYmo7dkij+OHhwKVLQLt2wJkzgJsb4O5OcbX0dKB1a8DTk967e5cSuXkMskkTwNUVqFULWL26aOM6d6ZAek4O7cPBgY7ZqRNw+jTtn8fvUlIKb9+gAcUa8/IAnY724+dH+5Lj7Ax4e9P+3dxopHXfPiAggPIZ9++n9UJD6Xw4/JzataPncXH02dSuTfbcvQs8/zzQtSvw/fcUO1SpACcnskejARITJVs9PSmu2rIlDWaFh5MNe/ZQfLRnT4qR6nQU/0xJAaZMoZisWm0ab9ZopLirvb2UImZnR/9bABg9mtoDRUVJ/xtfX4rHFhwIA4Bhw+jzyMoCZs6UPgM3N/r8XnyRPp+//qJ92NvTud65Q//LP/+kbbp1A0aMAMaNo8/orbdo3WXLCqfB8f8Pt6dePbL/9u3C6/n703dj3Tr6nzdsCFy8SO85OgJeXuRP3rpF30FbW0ms7O3p+/HoEa0bGkrxX34cf38qSZmZSTFizuzZwNq1wJEjgI8PfR49e9L/+X//o7/vvkvnvH8/8O230v+gUSP6fI4do///w4f0P2vQgL4Xly/T/yMhgX5bOh39JnkOd1CQNIgZEkLfnwMHpP/LCy/Q+RTMOHnjDYqhjx5dYfm6iqvuqdVq1KpVC+Hh4ejatSvCw8NRt27dsrLvieTm5sLBwQG//vor+vXrZ1w+ePBgpKSkWNQL0pqVsQQCgYBToVX3bt++jaioKNjb22P27NmoX78+atWqhcjISCxZsqRUxpQErVaLVq1aITo62rjMYDAgOjoaYfKWLgKBQFCFKXU96ytXrmDGjBlYsWIFDAYD9AXTd8qB1atXY/Dgwfj+++/Rtm1bzJkzB2vWrMHFixcLxbLNITxrgUBQFlhTWxTHrLOysnDgwAHs3bsXe/fuxYkTJ9CwYUOMGjUKXbt2LZUxJeW1117D/fv38dlnnyEhIQHNmzfH9u3bLRJqgUAgqAoo9qy1Wi3c3d0RGRmJrl27onPnznB3dy8r+8oF4VkLBIKyoEI96969e+PAgQNYtWoVEhISkJCQgK5du6J+/fqlMkQgEAgERaN4gHHDhg148OABtm/fjrCwMPzxxx/o3Lkz/P39ERkZWRY2CgQCwVOPYs+aExoaivz8fOTm5iI7Oxs7duzA6tWrsWLFCmvaVy7wSFCaqMshEAisCNeUUuZxACiBWH/99dfYu3cvDhw4gPT0dDRr1gxdunTBiBEj0Llz51IbVBGkP548oHQmo0AgEFhCeno6XF1dS7UPxQOMbdq0MU6I6dy5c6kNqAwYDAbcvXsXzs7OxTYD5vAZj7dv367yA5LiXCon1eVcqst5ACU7F8YY0tPT4efnB3Up6ykr9qyPHDlSqgNWRvisTKW4uLhU+S8gR5xL5aS6nEt1OQ9A+blYy6EVpfMFAoGgCiDEWiAQCKoAQqxLgE6nw+TJk6HT6SralFIjzqVyUl3OpbqcB1Dx51Lq2iACgUAgKHtKnGcNUOH/2NhY6PV6tGnTBjVr1rSWXQKBQCCQUWKx/u233zBs2DDUr18feXl5uHTpEhYsWIChQ4da0z6BQCAQQEEYJCMjA05OTsbXTZs2xa+//mqsCbJ161YMHz4cdwt2FBEIBAJBqbF4gLFVq1YmXVdsbGxw79494+vExERotVrrWicQCAQCAArEeseOHVi8eDFefvll3L17F3PnzsVrr70GX19feHl5YcKECfjuu+/K0tZKw4IFCxAUFAQ7Ozu0a9cOhw8frlB7oqKi0KZNGzg7O8Pb2xv9+vXDpUuXTNbJzs7GyJEj4enpCScnJ/Tv3x+JvJfhY27duoU+ffrAwcEB3t7eGD9+PPJ5n8HH7N27Fy1btoROp0PdunWxbNmyMjuvWbNmQaVSYcyYMVXyPO7cuYM33ngDnp6esLe3R2hoKI4ePWp8nzGGzz77DDVr1oS9vT169OiBK1eumOwjOTkZkZGRcHFxgZubG4YNG4aMjAyTdU6fPo3OnTvDzs4OAQEBmD17tlXPQ6/XY9KkSQgODoa9vT2eeeYZTJ8+3aTeRWU9l/3796Nv377w8/ODSqXChg0bTN4vT7vXrl2Lhg0bws7ODqGhodi2bZuyk1HaDn3lypWsbt26bN68eSwrK4udPn2anThxgj169KjUrdarAqtWrWJarZb973//Y+fOnWPDhw9nbm5uLDExscJs6tmzJ1u6dCk7e/YsO3nyJOvduzcLDAxkGRkZxnXeffddFhAQwKKjo9nRo0dZ+/btWYcOHYzv5+fnsyZNmrAePXqwEydOsG3btjEvLy82ceJE4zrXr19nDg4ObOzYsez8+fPs22+/ZRqNhm3fvt3q53T48GEWFBTEmjZtykaPHl3lziM5OZnVrl2bDRkyhMXGxrLr16+zHTt2sKtXrxrXmTVrFnN1dWUbNmxgp06dYi+99BILDg42+S316tWLNWvWjP3111/szz//ZHXr1mWDBg0yvp+amsp8fHxYZGQkO3v2LPvll1+Yvb09+/777612LjNmzGCenp5sy5YtLC4ujq1du5Y5OTmxuXPnVvpz2bZtG/vkk0/YunXrGAC2fv16k/fLy+6DBw8yjUbDZs+ezc6fP88+/fRTZmtry86cOWPxuSgWa8YYe/jwIRs2bBhr27YtO3nyZEl2UWVp27YtGzlypPG1Xq9nfn5+LCoqqgKtMuXevXsMANu3bx9jjLGUlBRma2vL1q5da1znwoULDACLiYlhjNGXWq1Ws4SEBOM6CxcuZC4uLiwnJ4cxxtjHH3/MGjdubHKs1157jfXs2dOq9qenp7N69eqxnTt3svDwcKNYV6Xz+Pe//806depU5PsGg4H5+vqy//znP8ZlKSkpTKfTsV9++YUxxtj58+cZAHbkyBHjOr///jtTqVTszp07jDHGvvvuO+bu7m48N37sBg0aWO1c+vTpw9566y2TZa+88gqLjIysUudSUKzL0+4BAwawPn36mNjTrl079s4771hsv6JJMdu2bcNXX32Fo0ePYsmSJZg9ezYiIyMxfvx4PHr0SJlLXwXJzc3FsWPH0KNHD+MytVqNHj16ICYmpgItMyU1NRUA4OHhAQA4duwY8vLyTOxu2LAhAgMDjXbHxMQgNDTUpBVaz549kZaWhnPnzhnXke+Dr2Ptcx85ciT69OlT6FhV6Tw2bdqE1q1b49VXX4W3tzdatGiBH374wfh+XFwcEhISTOxwdXVFu3btTM7Fzc0NrVu3Nq7To0cPqNVqxMbGGtfp0qWLyXhRz549cenSJTx8+NAq59KhQwdER0fj8uXLAIBTp07hwIEDeOGFF6rcucgpT7ut8Z2zWKzHjRuHoUOH4siRI3jnnXcwffp0hIeH4/jx47Czs0OLFi3w+++/W3zgqsiDBw+g1+sL9Xb08fFBQkJCBVllisFgwJgxY9CxY0c0adIEAJCQkACtVgs3NzeTdeV2JyQkmD0v/l5x66SlpVntYr1q1SocP34cUVFRhd6rSudx/fp1LFy4EPXq1cOOHTvwr3/9Cx988AGWL19uYktx36WEhAR4e3ubvG9jYwMPDw9F51taJkyYgIEDB6Jhw4awtbVFixYtMGbMGGOzkap0LnLK0+6i1lFyXhbnWS9btgx//PEHWrVqheTkZLRv3x6TJk2CVqvF9OnTMWjQILzzzjvGq62gYhg5ciTOnj2LAwcOVLQpirl9+zZGjx6NnTt3ws7OrqLNKRUGgwGtW7fGzJkzAQAtWrTA2bNnsWjRIgwePLiCrVPGmjVrsGLFCqxcuRKNGzfGyZMnMWbMGPj5+VW5c6nKWOxZOzo6Ii4uDgD9qAr+mBo1aoQ///zTutZVMry8vKDRaAplHyQmJsLX17eCrJIYNWoUtmzZgj179piUfPX19UVubi5SUlJM1pfb7evra/a8+HvFrePi4gJ7e/tS23/s2DHcu3cPLVu2hI2NDWxsbLBv3z7MmzcPNjY28PHxqRLnAQA1a9ZEo0aNTJaFhITg1q1bJrYU913y9fU1SY8FgPz8fCQnJys639Iyfvx4o3cdGhqKN998Ex9++KHx7qcqnYuc8rS7qHWUnJfFYh0VFYV//vOf8PPzQ3h4OKZPn27xQaoLWq0WrVq1QnR0tHGZwWBAdHQ0wsLCKswuxhhGjRqF9evXY/fu3QgODjZ5v1WrVrC1tTWx+9KlS7h165bR7rCwMJw5c8bki7lz5064uLgYRScsLMxkH3wda5179+7dcebMGZw8edL4aN26NSIjI43Pq8J5AEDHjh0LpU9evnwZtWvXBgAEBwfD19fXxI60tDTExsaanEtKSgqOHTtmXGf37t0wGAxo166dcZ39+/cjLy/P5FwaNGgAd3d3q5xLVlZWocL5Go0GBoOhyp2LnPK02yrfOYuHIhljDx48YIcPH2YPHz5Uslm1YtWqVUyn07Fly5ax8+fPsxEjRjA3NzeT7IPy5l//+hdzdXVle/fuZfHx8cZHVlaWcZ13332XBQYGst27d7OjR4+ysLAwFhYWZnyfp7w9//zz7OTJk2z79u2sRo0aZlPexo8fzy5cuMAWLFhQZql7HHk2SFU6j8OHDzMbGxs2Y8YMduXKFbZixQrm4ODAfv75Z+M6s2bNYm5ubmzjxo3s9OnTLCIiwmzaWIsWLVhsbCw7cOAAq1evnknaWEpKCvPx8WFvvvkmO3v2LFu1ahVzcHCwaure4MGDmb+/vzF1b926dczLy4t9/PHHlf5c0tPT2YkTJ9iJEycYAPb111+zEydOsJs3b5ar3QcPHmQ2Njbsyy+/ZBcuXGCTJ08un9S9p51vv/2WBQYGMq1Wy9q2bcv++uuvCrUHgNnH0qVLjes8evSIvffee8zd3Z05ODiwl19+mcXHx5vs58aNG+yFF15g9vb2zMvLi40bN47l5eWZrLNnzx7WvHlzptVqWZ06dUyOURYUFOuqdB6bN29mTZo0YTqdjjVs2JAtXrzY5H2DwcAmTZrEfHx8mE6nY927d2eXLl0yWScpKYkNGjSIOTk5MRcXFzZ06FCWnp5uss6pU6dYp06dmE6nY/7+/mzWrFlWPY+0tDQ2evRoFhgYyOzs7FidOnXYJ598YpKqVlnPZc+ePWZ/G4MHDy53u9esWcPq16/PtFota9y4Mdu6dauicxElUgUCgaAKIJoPCAQCQRVAiLVAIBBUAYRYCwQCQRVAiLVAIBBUAYRYCwQCQRVAiLVAIBBUAYRYCwQCQRVAiLXgqSErKwv9+/eHi4sLVCpVoRojADBlyhQ0b9683G17El27djXpmCN4+hBiLSgzhgwZApVKhVmzZpks37BhA1QqVbnbs3z5cvz55584dOgQ4uPj4erqWmidjz76yKSGw5AhQ9CvX79ys3Hv3r1mLyTr1q17KuvxCCSEWAvKFDs7O3zxxRdlUjxeKdeuXUNISAiaNGkCX19fsxcMJycneHp6Wv3Yubm5pdrew8MDzs7OVrJGUBURYi0oU3r06AFfX1+zzQTk/Pbbb2jcuDF0Oh2CgoLw1VdfKT5Wcfvo2rUrvvrqK+zfvx8qlQpdu3Y1uw95GGTKlClYvnw5Nm7cCJVKBZVKhb179wKgMsEDBgyAm5sbPDw8EBERgRs3bhj3wz3yGTNmwM/PDw0aNAAA/PTTT2jdujWcnZ3h6+uL119/3Vgh8MaNG+jWrRsAwN3dHSqVCkOGDDHaLw+DPHz4EP/85z/h7u4OBwcHvPDCCyaNXpctWwY3Nzfs2LEDISEhcHJyQq9evRAfH29cZ+/evWjbti0cHR3h5uaGjh074ubNm4o/d0H5IMRaUKZoNBrMnDkT3377Lf7++2+z6xw7dgwDBgzAwIEDcebMGUyZMgWTJk1S1HH8SftYt24dhg8fjrCwMMTHx2PdunVP3OdHH32EAQMGGEUuPj4eHTp0QF5eHnr27AlnZ2f8+eefOHjwoFEM5R50dHQ0Ll26hJ07d2LLli0AgLy8PEyfPh2nTp3Chg0bcOPGDaMgBwQE4LfffgNApV/j4+Mxd+5cs7YNGTIER48exaZNmxATEwPGGHr37m1SpjMrKwtffvklfvrpJ+zfvx+3bt3CRx99BIBqMvfr1w/h4eE4ffo0YmJiMGLEiAoJTwksRFHZJ4FAAYMHD2YRERGMMcbat29vbLq6fv16Jv/qvf766+y5554z2Xb8+PGsUaNGFh/Lkn2MHj2ahYeHF7ufyZMns2bNmpk9B85PP/3EGjRowAwGg3FZTk4Os7e3Zzt27DBu5+PjY1KZzhxHjhxhAIxV3HiVuIJliOXVBy9fvswAsIMHDxrff/DgAbO3t2dr1qxhjDG2dOlSBsCkm/qCBQuYj48PY4wqyQFge/fuLdY+QeVBeNaCcuGLL77A8uXLceHChULvXbhwAR07djRZ1rFjR1y5cgV6vd6i/VtjH5Zy6tQpXL16Fc7OznBycoKTkxM8PDyQnZ2Na9euGdcLDQ01aaIK0B1A3759ERgYCGdnZ4SHhwOAsYOMJVy4cAE2NjbG4vcA4OnpiQYNGph8vg4ODnjmmWeMr2vWrGkMuXh4eGDIkCHo2bMn+vbti7lz55qESASVDyHWgnKhS5cu6NmzJyZOnFjRppSajIwMtGrVyqSjzcmTJ3H58mW8/vrrxvUcHR1NtsvMzETPnj3h4uKCFStW4MiRI1i/fj2A0g9AmsPW1tbktUqlApNVRF66dCliYmLQoUMHrF69GvXr18dff/1ldTsE1sHihrkCQWmZNWsWmjdvbhxs44SEhODgwYMmyw4ePIj69etDo9FYtG9r7MMcWq22kGfesmVLrF69Gt7e3nBxcbF4XxcvXkRSUhJmzZqFgIAAAMDRo0cLHQ9AsXcDISEhyM/PR2xsLDp06AAASEpKwqVLlwr1fXwSLVq0QIsWLTBx4kSEhYVh5cqVaN++vaJ9CMoH4VkLyo3Q0FBERkZi3rx5JsvHjRuH6OhoTJ8+HZcvX8by5csxf/5842AYQP0Z58+fX+S+LdlHSQgKCsLp06dx6dIlPHjwAHl5eYiMjISXlxciIiLw559/Ii4uDnv37sUHH3xQ5CAqAAQGBkKr1eLbb7/F9evXsWnTpkK507Vr14ZKpcKWLVtw//59ZGRkFNpPvXr1EBERgeHDh+PAgQM4deoU3njjDfj7+yMiIsKi84qLi8PEiRMRExODmzdv4o8//sCVK1cQEhKi7AMSlBtCrAXlyrRp04yNVjktW7bEmjVrsGrVKjRp0gSfffYZpk2bZsySAChH+sGDB0Xu15J9lIThw4ejQYMGaN26NWrUqIGDBw/CwcEB+/fvR2BgIF555RWEhIRg2LBhyM7OLtbTrlGjBpYtW4a1a9eiUaNGmDVrFr788kuTdfz9/TF16lRMmDABPj4+GDVqlNl9LV26FK1atcKLL76IsLAwMMawbdu2QqGPonBwcMDFixfRv39/1K9fHyNGjMDIkSPxzjvvWP7hCMoV0dZLIBAIqgDCsxYIBIIqgBBrgUAgqAIIsRYIBIIqgBBrgUAgqAIIsRYIBIIqgBBrgUAgqAIIsRYIBIIqgBBrgUAgqAIIsRYIBIIqgBBrgUAgqAIIsRYIBIIqgBBrgUAgqAL8P785ZS2IapiaAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 360x270 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mean_32 = np.mean(const_32, axis = 0)\n",
    "mean_16 = np.mean(const_16, axis = 0)\n",
    "stdev_32 = np.std(const_32, axis = 0)\n",
    "stdev_16 = np.std(const_16, axis = 0)\n",
    "\n",
    "x = [i for i in range(len(mean_32))]\n",
    "#plt.figure(figsize=(4.8,3.6))\n",
    "plt.figure(figsize=(3.6,2.7))\n",
    "plt.xlabel('No. of iterations')\n",
    "plt.ylabel('% weights stuck at constant values')\n",
    "plt.plot(x, mean_32, label = \"Float32\", color = 'red')\n",
    "plt.plot(x, mean_16, label = \"Float16\", color = 'blue')\n",
    "print((mean_32 - stdev_32).shape)\n",
    "print(np.full(10000, 0).shape)\n",
    "plt.fill_between(x, np.maximum(mean_32 - stdev_32, np.full(10000, 0)), np.minimum(mean_32 + stdev_32, np.full(10000, 100)), color = 'red', alpha = 0.3)\n",
    "plt.fill_between(x, np.maximum(mean_16 - stdev_16, np.full(10000, 0)), np.minimum(mean_16 + stdev_16, np.full(10000, 100)), color = 'blue', alpha = 0.3)\n",
    "leg1 = plt.legend(loc = 'right', frameon=False)\n",
    "plt.savefig(\"constant-weights.pdf\", bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10000,)\n",
      "(10000,)\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAEPCAYAAACwWiQoAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABBgElEQVR4nO3deVxUVf8H8M+wzLDOsCibQiJuqLiihJpa8kRlhumTZZRYLj2GKZr6aKXlkpC5m0maqfW4pCVW/kwzFFFCFFxJAxdUlE1FGAGBYeb8/rjeYQaG9c4MMH7fr9e8hrn3zLnnDsNnDufeuUfEGGMghBDSopk1dQMIIYQIR2FOCCEmgMKcEEJMAIU5IYSYAApzQggxARTmhBBiAijMCSHEBFCYE0KICbBo6gY0dyqVCllZWbC3t4dIJGrq5hBCTARjDA8fPoSHhwfMzIT3qynM65CVlQVPT8+mbgYhxERlZmaibdu2guuhMK+Dvb09AO4Fl0qlTdwaQoipkMvl8PT0VGeMUBTmdeCHVqRSKYU5IUTv9DV8SwdACSHEBFCYE0KICWjWYR4fH48RI0bAw8MDIpEI+/bt01rPGMOCBQvg7u4Oa2trBAUF4cqVK1pl8vPzERoaCqlUCgcHB0yYMAFFRUVG3AtCCDG8Zh3mxcXF6NmzJ9avX69z/bJly7B27VpER0cjKSkJtra2CA4ORmlpqbpMaGgo/v77bxw+fBj79+9HfHw8Jk+ebKxdIIQQ42AtBAAWExOjfqxSqZibmxv78ssv1csKCgqYRCJhO3fuZIwxdunSJQaAnT59Wl3m999/ZyKRiN25c6de2y0sLGQAWGFhoX52hBBCmP6zpVn3zGuTkZGBnJwcBAUFqZfJZDIEBAQgMTERAJCYmAgHBwf4+/urywQFBcHMzAxJSUk66y0rK4NcLte6EUJIc9diT03MyckBALi6umotd3V1Va/LycmBi4uL1noLCws4OTmpy1QVGRmJhQsX6qWNjx4BR44AxcWAHr7gRQhpxlQqwNcX8PNrmu232DA3lHnz5mHmzJnqx/yJ/Y1RUgLcuAEolYBEoqcGEkKapdu3gSp9S6NqsWHu5uYGAMjNzYW7u7t6eW5uLnr16qUuk5eXp/W8iooK5Ofnq59flUQigURPyatSAYwBHh4U5oSYOo3zLppEi/3n39vbG25uboiNjVUvk8vlSEpKQmBgIAAgMDAQBQUFSElJUZc5cuQIVCoVAgICDN5GxrgbXZ+LEGJozbpnXlRUhKtXr6ofZ2Rk4Ny5c3BycoKXlxciIiKwZMkSdOzYEd7e3pg/fz48PDwwcuRIAICvry9eeOEFTJo0CdHR0VAoFJg6dSreeOMNeHh4GLz9FOaEEGNp1mGenJyMZ599Vv2YH8sOCwvD1q1bMWfOHBQXF2Py5MkoKCjAoEGDcPDgQVhZWamfs337dkydOhXDhg2DmZkZRo8ejbVr1xql/XyYE0KIoYkYo7ipjVwuh0wmQ2FhYYMvtJWVBezeDTz1FGDRrD82CSFCXb8O9OoFDBlSv/JCskWXFjtm3hLwH5M0zEIIMTQKcwNijDujhcKcEGJoFOYGpFJx9xTmhBBDozA3MDqbhRBiDBTmBsT3zAkhxNAozA2IzhMihBgLhbkBUZgTQoyFwtyAKMwJIcZCYW5AFOaEEGOhMDcgCnNCiLFQmBsQnc1CCDEWCnMDop45IcRYKMwJIcQEUJgbEA2zEEKMhcLcgGiYhRBiLBTmhBBiAijMDYiGWQghxkJhbkA0zEIIMRYKcwOiMCeEGAuFOSGEmAAKcwOiMXNCiLFQmBsQDbMQQoyFwtyAKMwJIcYiOMwfPXqEkpIS9eObN29i9erV+OOPP4RW3eJRmBNCjEVwmIeEhOD7778HABQUFCAgIAArVqxASEgINmzYILiBLRmFOSHEWASH+ZkzZ/DMM88AAH766Se4urri5s2b+P7777F27VrBDSSEEFI3wWFeUlICe3t7AMAff/yBUaNGwczMDE8//TRu3rwpuIEtmUoFiERN3QpCyJNAcJh36NAB+/btQ2ZmJg4dOoTnn38eAJCXlwepVCq4gS0ZnZpICDEWwWG+YMECzJo1C+3atUNAQAACAwMBcL303r17C25gbZRKJebPnw9vb29YW1vDx8cHixcvBtMYrGaMYcGCBXB3d4e1tTWCgoJw5coVg7arctvUMyeEGIeF0Ar+/e9/Y9CgQcjOzkbPnj3Vy4cNG4ZXX31VaPW1+uKLL7BhwwZs27YN3bp1Q3JyMt555x3IZDJMmzYNALBs2TKsXbsW27Ztg7e3N+bPn4/g4GBcunQJVlZWBm0fDbMQQoxFcJjfunULnp6ecHNz01rer18/ZGZmCq2+Vn/99RdCQkIwfPhwAEC7du2wc+dOnDp1CgDXK1+9ejU++eQThISEAAC+//57uLq6Yt++fXjjjTcM2j4aZiGEGIvgYRZvb2/cvXu32vL8/Hx4e3sLrb5WAwYMQGxsLNLT0wEA58+fx4kTJ/Diiy8CADIyMpCTk4OgoCD1c2QyGQICApCYmKizzrKyMsjlcq1bY9GpiYQQYxHcM2eMQaRjLKGoqMjgwxhz586FXC5Hly5dYG5uDqVSic8//xyhoaEAgJycHACAq6ur1vNcXV3V66qKjIzEwoUL9dI+lQowo+/YEkKMoNFhPnPmTACASCTC/PnzYWNjo16nVCqRlJSEXr16CW5gbXbv3o3t27djx44d6NatG86dO4eIiAh4eHggLCysUXXOmzdPvW8AIJfL4enp2ai6aJiFEGIsjQ7zs2fPAuB65hcvXoRYLFavE4vF6NmzJ2bNmiW8hbWYPXs25s6dqx779vPzw82bNxEZGYmwsDD1OH5ubi7c3d3Vz8vNza3xg0YikUAikeilfTTMQggxlkaH+dGjRwEA77zzDtasWdMk55SXlJTArMo4hrm5OVSPu8Te3t5wc3NDbGysOrzlcjmSkpIwZcoUg7ePzmYhhBiL4DHzLVu26KMdjTJixAh8/vnn8PLyQrdu3XD27FmsXLkS7777LgBuCCgiIgJLlixBx44d1acmenh4YOTIkQZvH51nTggxFsFhDgCxsbGIjY1FXl6eulfM++677/SxCZ3WrVuH+fPn4/3330deXh48PDzw3nvvYcGCBeoyc+bMQXFxMSZPnoyCggIMGjQIBw8eNPjBWYB65oQQ4xExJmxkd+HChVi0aBH8/f3h7u5e7cyWmJgYQQ1sanK5HDKZDIWFhQ0eSjpwAMjIALy8DNQ4Qkizcf060KsXMGRI/coLyRZdBPfMo6OjsXXrVrz99tuCG2Nq6AAoIcRYBJ8FXV5ejgEDBuijLSaHxswJIcYiOMwnTpyIHTt26KMtJofGzAkhxtKoYRbNL9WoVCps3LgRf/75J3r06AFLS0utsitXrhTWwhaMhlkIIcbSqDDnvzDE48/hTk1NFdwgU0JhTggxlkaFOf+FIVI7GmYhhBiL4LNZNIdcNIlEIlhZWaFDhw4ICQmBk5OT0E21OBTmhBBjERzmZ8+exZkzZ6BUKtG5c2cAQHp6OszNzdGlSxd8/fXX+PDDD3HixAl07dpVcINbEhpmIYQYi+CzWUJCQhAUFISsrCykpKQgJSUFt2/fxr/+9S+MHTsWd+7cweDBgzFjxgx9tLfFoZ45IcQYBIf5l19+icWLF2t9g0kmk+Gzzz7DsmXLYGNjgwULFiAlJUXoplocugQuIcRYBId5YWEh8vLyqi2/e/euepYeBwcHlJeXC91Ui0Nj5oQQY9HLMMu7776LmJgY3L59G7dv30ZMTAwmTJigvjLhqVOn0KlTJ6GbanFozJwQYiyCD4B+8803mDFjBt544w1UVFRwlVpYICwsDKtWrQIAdOnSBd9++63QTbU49HV+QoixCA5zOzs7bNq0CatWrcL169cBAO3bt4ednZ26jKGnj2uuaJiFEGIsermeOcCFeo8ePfRVnUlgDLh1i7sU7tixgJ5moyOEkGoafW2WxYsXw9bWtsYvDfGe1Guz8OPlixdzPysUwKRJTdsmQojpavS1WRQKhfrnmlSdqOJJwhhQVlYZ6gkJFOaEEMMRfG0Wuk6LbowBt29XPi4oAORyoAnmvSaEPAEEn5pIdGMMuH+/8rFKBaxZA9y923RtIoSYLr0cAD1+/Di++eYbXLt2DT/99BPatGmDH374Ad7e3hg0aJA+NtHiMAbk52svO30ayMsD1q2rXFZWBqSmVvbcLS2BNm0ABwfA2hpwduaWEUJIbQSH+c8//4y3334boaGhOHv2LMrKygBw3wxdunQpDhw4ILiRLRFjQHEx93OvXkBaGvDoEXDzJnDnDhfYSiUwdy5w7Vrtdbm6Ap06Ad26AY6OgEzGhbyTEwU9IYQjOMyXLFmC6OhojBs3Drt27VIvHzhwIJYsWSK0+haLPwAKAN7ewKJFwPz5wPnzwNGjQGgocO4cF+Tm5kDPnoC9Pdc7z84GSku5n1UqIDeXux0/Xn07traAiwsX6rm5wMiRXNDb23M9e6kUsLICLCy4m5kZUFjI/ZdQUMAts7bm7s3NuRtjQEVF5Rg//7ydO4Hx47n9MTPj2qZUcuUZA8Tihp9XX1d5fRxDbw7bqKtMeTn3exKLK5fxr6vmN4lre6x5r2sZABw5Aly9CrRqxS1TKrnfdUUF14acHMDGpu59MRYhvxulsnJfJRLuPZuZya1r35671/WaVX1c28+ayxQK4IUXgCFDGt9mIQSHeVpaGgYPHlxtuUwmQ0FBgdDqWyzGuEAGuD9SAOjXjwvz3bu5s1vu3OGWv/AC8N571etQKrngvXYNuHSJuy8q4sbiCwu5MC0uBjIyKp+zbZth92vFCsPWT4i+3btXfdnj7zfqXWGhYeqtD8Fh7ubmhqtXr6Jdu3Zay0+cOIH2/MffE0ilquyZ82H+3HPAiRPAlSuVQW5hAYwYobsOc3NuKMXJifsg0KRUckF+7x43Dp+bC5SUcGfQ5OdX9uxLSrh2PL7SgnqbPj7c0E1ZGXdTKitv/DdXray4x48eAZpnoNrbc/ciEdfbMXt8GP3x2aoA6r4ujZDr1jRV3YbaLv+614bvoWrea/Zaa3pcdVlJCfd79/fnfm/m5pX/lRUWVg7fCaGvaxLpo578fMDOjtsvlYrrFJWWAt27c+s1X6O6XsOqP1e9z8oC+vcX3ubGEhzmkyZNwvTp0/Hdd99BJBIhKysLiYmJmDVrFubPn6+PNrZImj1za2vu3s4OWLaM610fPQr88w8QHAx4eDS8fnNzbghEKq38l7Gu9lRUcG/oxgyHANx/BI6OleFN9IsP9JrChQjn52e4ui0tAXd3w9VfF8FhPnfuXKhUKgwbNgwlJSUYPHgwJBIJZs2ahQ8++EAfbWyRNMfM+Z45z86O643X1CM3BJFI+MFSZ2f9tIXoZm7e1C0gLZngMBeJRPj4448xe/ZsXL16FUVFRejatavWhbaeRLrGzAkhxFAEh/m4cePw7LPPYsiQIU/cHJ+10eyZ88MshBBiKIJHP8ViMSIjI9GhQwd4enrirbfewrfffosrV67oo311unPnDt566y04OzvD2toafn5+SE5OVq9njGHBggVwd3eHtbU1goKCjNI2zTCnqyUSQgxNcJh/++23SE9PR2ZmJpYtWwY7OzusWLECXbp0Qdu2bfXRxho9ePAAAwcOhKWlJX7//XdcunQJK1asgKOjo7rMsmXLsHbtWkRHRyMpKQm2trYIDg5GKT8GYiD8eacAhTkhxPD0dj1zR0dHODs7w9HREQ4ODrCwsEDr1q31Vb1OX3zxBTw9PbFlyxb1Mm9vb/XPjDGsXr0an3zyCUJCQgAA33//PVxdXbFv3z688cYbBmubSlV5OiB9S5MQYmiCe+YfffQRBgwYAGdnZ8ydOxelpaWYO3cucnJyar08rj78+uuv8Pf3x2uvvQYXFxf07t0bmzZtUq/PyMhATk4OgoKC1MtkMhkCAgKQmJho0LbxpwIC3Hm8hJCG++ijodi0KaKpm9EiCI6ZqKgotG7dGp9++ilGjRpl1Imbr1+/jg0bNmDmzJn46KOPcPr0aUybNg1isRhhYWHIyckBALi6umo9z9XVVb2uqrKyMvX1ZQBALpc3qm2aYU49c0Jqtnr1eBw5Uv2ry9HRhjm29dFHQ+Ht3QuTJq1WL5PL72PFilDcvHkBcvl9ODi4oH//EIwbtxQ2Ntx1q//6ay9+/30DMjLOQaEog5dXN4wd+xn69Ak2SDsbSnCYnz17FseOHUNcXBxWrFgBsViMIUOGYOjQoRg6dKhBw12lUsHf3x9Lly4FAPTu3RupqamIjo5GWFhYo+qMjIzEwoULBbeN/4IOQGFOSF369HkB06dv0VomlRp2mFaTmZkZAgJC8NZbSyCTtUZ29lVER4fj66/zMWvWDgDA33/Ho1evf2HcuKWwtXXAn39uwZIlI/Dll0nw8elttLbWRHCY9+zZEz179sS0adMAAOfPn8eqVasQHh4OlUoFZV3fUxbA3d292umQvr6++PnnnwFwlxoAgNzcXLhrfDUrNze3xkmm582bpzUVnlwuh6enZ4Pbpnl8lcKckNpZWkrg6OhWZ7miogfYtGk6Tp36DQpFGbp3H4LJk9fCw6MjAK6H/c03U/H33/EoKnoAd3cf/PvfH2HIkLEAuP8CUlOPITX1GH77bQ0AYNOmDLi6tsNLL01Rb8fF5Sm89NL7iIn5Ur1MsycPAOPGLUVS0i84ffo30whzxhjOnj2LuLg4xMXF4cSJE5DL5ejRoweGGPjyYQMHDkRaWprWsvT0dDz11FMAuIOhbm5uiI2NVYe3XC5HUlISpkyZUrU6AIBEIoFED6efaIzUUJiTpsEYzMtKjL5ZpcTGYNchWL16PLKzr+CTT36FjY0UW7f+FwsXvoT16y/BwsISCkUpOnToi9Gj/wsbGymSk/8Pq1a9DXd3H3Tq1B+TJq1BVlY6vLy6IzR0EQDd/wHcv5+FxMS96Nat5gxTqVR49Ogh7OwEXsxGTwSHuZOTE4qKitCzZ08MGTIEkyZNwjPPPAMHBwc9NK92M2bMwIABA7B06VKMGTMGp06dwsaNG7Fx40YA3LdTIyIisGTJEnTs2BHe3t6YP38+PDw8MHLkSIO2jQ9zkYi+pk2ahnlZCV4aY/xvYh/YXQSllW2DnnP69H6M0Whrnz4vYu7cPVplsrKu4NSpX/HFFwnw9R0AAPjww+14911PnDy5D4MGvQZn5zZ49dVZ6ue8/PIHOHPmEE6c2I1OnfrD1lYGCwsxJBIbnf8JfPnlWCQl/YLy8kfo338EPvjg2xrbHBOzHKWlRRg0aEyD9tVQBIf5//73PzzzzDOQNsHklv369UNMTAzmzZuHRYsWwdvbG6tXr0ZoaKi6zJw5c1BcXIzJkyejoKAAgwYNwsGDB2Fl4O/Y82FuYUEXSyKkLn5+z2LKlA3qx1Y6PgwyMy/D3NwCnToFqJdJpc5o06Yzbt++DABQKpXYs2cpEhJ24/79O6ioKIdCUQaJpH4XaZ84cRXGjv0Ud+6k4/vv52Hz5pmYMuXrauWOHduBXbsW4uOPf4GDg0tDd9cgBIf58OHD9dGORnv55Zfx8ssv17heJBJh0aJFWLRokRFbVTlmTkMspKkoJTY4sLuoSbbbUFZWtvDw6CB42zExX+K339Zg4sTVaNfODxKJLb79NgIVFeX1er6joxscHd3Qtm0X2Ns7Ye7cZ/D66/Ph5FR5zC0+fhfWrZuI//53D3r1CqqlNuOiM6ANRLNnTkiTEIkaPNzRnHl6+kKprEB6epJ6mEUuv487d9Lg6cmdCHH5cgICAkLw7LNvAeDGtbOy0tXrAcDCQgyVqu4TM1SPT0dTKCoPgB07thPr1r2LWbN2oV+/pu3IVkVRYyDUMydEvzw8OiIgIARffTUJ4eHfwNraHtu2zYWzcxsEBISoyyQk/ITLl/+CnZ0jfvllJQoKcrXC3MWlHdLTk5CbewPW1naws3PCmTMHUVCQi44d+8HKyg63bv2NrVtnw9d3IFxd2wHghlZWrw7DpElr0LlzAB484L6rIhZbw9ZWZvTXo6pGhfmFCxfQvXt3mNEsBTXie+YU5oToz/TpW7Bp03QsXvwyFIpydOs2GJ9+egAWFtwf2pgxnyAn5zo++ywYYrENgoMnIyBgJEpKKudze/XVWVi9Ogzh4V1RXv4ImzZlQCy2xh9/bMLmzTOgUJShVStPBAaOwujRc9XPO3RoI5TKCkRHhyM6Oly9/LnnwhARsdVor0FNRIw1fHImc3NzZGdnw8XFBe3bt8fp06fhbKIzF8jlcshkMhQWFjboIO/OncCbbwKensD69QZsICGkWbh+HejVq/4TOjc2W2rSqK61g4MDMh7PInzjxg312BKpRD1zQogxNWqYZfTo0RgyZAjc3d0hEong7+8P8xpOpr5uqGmwmzkaMyeEGFOjwnzjxo0YNWoUrl69imnTpmHSpEmw56dsJwCA8sdnQlGYE0KModFns7zwwgsAgJSUFEyfPp3CvAoaZiGEGJPgUxM1J4YglSjMCSHGpJfzzAsKCrB582Zcvsx9pbZr166YMGECZLKmP/eyqfBTxtGXhgghxiD4RPHk5GT4+Phg1apVyM/PR35+PlatWgUfHx+cOXNGH21skfgxcwpzQogxCI6aGTNm4JVXXsGmTZtg8Ti5KioqMHHiRERERCA+Pl5wI1si6pkTQoxJcNQkJydrBTkAWFhYYM6cOfD39xdafYtFYU4IMSbBwyxSqRS3bt2qtjwzM/OJPsOFhlkIEY4mdK4/wVHz+uuvY8KECVi+fDkGDOCuZJaQkIDZs2dj7NixghvYUvE9c5qYgpDaNYcJnQFg48ZpuHw5ATdvpsLT0xdr1pyr9lzGGPbtW4FDhzYiL+8mpNJWeOml9zFmzMcGaWtDCA7z5cuXQyQSYdy4cah4PB29paUlpkyZgqioKMENbKlomIWQ+mvqCZ15QUHvIj09CTduXNC5ftOm6Th79g+8885yPPWUH4qK8vHwYb6RW6mb4KgRi8VYs2YNIiMjce3aNQCAj48PbGwafoF6U0JhTkj9NYcJnSdPXgsAKCy8qzPMMzMv4/ffN2DdulS0bdv58VJvPey9fugtamxsbODn56ev6lo8CnPS1BjTnljcWCQSw02VaKwJnXU5deo3uLm1x+nT+/HZZy8AYOjZMwjjxy+DvX3TT+pMUWMgNGZOmlpZGTCmCeYa3r0baOgUu81lQufa5OZeR17eTSQk7MGMGd9DpVLi229nICrq3/j88yMN22EDoDA3EOqZE1J/zWVC59qoVCooFGWYMeN7tGnTCQAwbdpmzJjRF7dvpwHoXHsFBkZRYyCPjwVTmJMmI5FwveSm2G5DNZcJnWvj5OQOc3MLdZADQNu2vgCAu3dvQSajMDdJ1DMnTU0kavhwR3Nm7Amdq/L1HQilsgLZ2dfg7u4DAMjKSgcAuLg81STHJzTpJWpKS0tx4cIF5OXlVZt16JVXXtHHJloc6pkTol+GnNDZzMwMWVlXUVpahIKCHJSXP8L16+cAAJ6eXWFpKUbPnkHw8emDtWvfxcSJq8GYCtHR4ejV619o06YTmnoeHsFRc/DgQYwbNw737t2rtk4kEkGpbPgnoCngvwFKB0AJ0R9DTejs6toOX301Eampx9TlIiJ6A6g8ddHMzAyffPIbNm78AB99NBgSiS369n0R7767wrgvQg0aNaGzpo4dO+L555/HggUL4Orqqq92NRuNnXTV3x9ISQE+/LD+E7wSQlquFjmhs6bc3FzMnDnTJINcCBozJ4QYk+Aw//e//424uDg9NMW08GPmNMxCCDEGwf3Gr776Cq+99hqOHz8OPz8/WFaZJ23atGlCN9Ei8WFO08YRQoxBcJjv3LkTf/zxB6ysrBAXFweRxvd4RSKRUcM8KioK8+bNw/Tp07F69WoA3Jk2H374IXbt2oWysjIEBwfj66+/NviwEH0DlBBiTIKHWT7++GMsXLgQhYWFuHHjBjIyMtS360Y8V+f06dP45ptv0KNHD63lM2bMwG+//YY9e/bg2LFjyMrKwqhRowzeHjo1kRBiTILDvLy8HK+//jrMzARX1WhFRUUIDQ3Fpk2b4OjoqF5eWFiIzZs3Y+XKlXjuuefQt29fbNmyBX/99RdOnjxp0DbRAVBCiDEJTuCwsDD8+OOP+mhLo4WHh2P48OEICgrSWp6SkgKFQqG1vEuXLvDy8kJiYqJB28SfXk9hTggxBsFRo1QqsWzZMhw6dAg9evSodgB05cqVQjdRq127duHMmTM4ffp0tXU5OTkQi8VwcHDQWu7q6oqcnByd9ZWVlaFM43u5crm8Ue2iMXNCiDEJDvOLFy+id2/um1Kpqala60SGuqjxY5mZmZg+fToOHz4MKz1dhCIyMhILFy4UXA+dzUIIMSbBYX706FF9tKNRUlJSkJeXhz59+qiXKZVKxMfH46uvvsKhQ4dQXl6OgoICrd55bm4u3Nx0X8t43rx5mDlzpvqxXC6Hp6dng9tG55kTQoxJ0Ji5QqHAsGHDcOWKYSZercuwYcNw8eJFnDt3Tn3z9/dHaGio+mdLS0vExsaqn5OWloZbt24hMDBQZ50SiQRSqVTr1hjUMyeEGJOgnrmlpSUuXNA98akx2Nvbo3v37lrLbG1t4ezsrF4+YcIEzJw5E05OTpBKpfjggw8QGBiIp59+2mDtYoxOTSSEGJfgs1neeustbN68WR9tMYhVq1bh5ZdfxujRozF48GC4ublh7969Bt0mf/AToDAnhBiH4KipqKjAd999hz///BN9+/aFra32dE+GPpulqqrXibGyssL69euxfv16o7WhXGNSExpmIYQYg+AwT01NVR+ATE9P11pn6LNZmivNGUeoZ04IMYYWfTZLc8X3zEUiOpuFEGIceuk3FhQUYPPmzbh8mZshu1u3bnj33Xchk8n0UX2Lw4c59coJIcYi+ABocnIyfHx8sGrVKuTn5yM/Px8rV66Ej48Pzpw5o482tjg0ZRwhxNgE9x1nzJiBV155BZs2bYLF465oRUUFJk6ciIiICMTHxwtuZEtDPXNCiLEJjpvk5GStIAcACwsLzJkzB/7+/kKrb5GoZ04IMTbBwyxSqRS3bt2qtjwzMxP29vZCq2+RqGdOCDE2wWH++uuvY8KECfjxxx+RmZmJzMxM7Nq1CxMnTsTYsWP10cYWhz81kcKcEGIsguNm+fLlEIlEGDduHCoef4fd0tISU6ZMQVRUlOAGtkTUMyeEGJvguBGLxVizZg0iIyNx7do1AICPjw9sbGwEN66lojFzQoix6a3vaGNjAz8/P31V16JRz5wQYmx6iZvY2FjExsYiLy8PKpVKa913332nj020KBTmhBBjExw3CxcuxKJFi+Dv7w93d/cn9nosmijMCSHGJjhuoqOjsXXrVrz99tv6aI9JoDAnhBib4FMTy8vLMWDAAH20xWTwpybSAVBCiLEIDvOJEydix44d+miLyaCeOSHE2ATHTWlpKTZu3Ig///wTPXr0gGWV2RiMPTlFc0BhTggxNsFxc+HCBfTq1QsAN1GFpif1YCgf5jTLECHEWGhyCgOgnjkhxNgEj5mT6ugboIQQY6MwNwCFgrs3o1eXEGIkFDcGoFRy9xTmhBBjobgxgMcXj6RhFkKI0VCYGwD1zAkhxtbguHn06BHu3LlTbfnff/+tlwaZgqo984cPgQcPmq49hBDT16Aw/+mnn9CxY0cMHz4cPXr0QFJSknodXZulUtWeeWYmkJdXeZYLIYToW4PCfMmSJUhJScG5c+ewZcsWTJgwQf1VfsaYQRrYEmmezVJRAUgkgKsrkJ/ftO0ihJiuBn2tRaFQwNXVFQDQt29fxMfH49VXX8XVq1ef2G976qLZMy8tBaysgNatgZMnuaGX1q2btn2EENPToJ65i4sLLly4oH7s5OSEw4cP4/Lly1rLjSUyMhL9+vWDvb09XFxcMHLkSKSlpWmVKS0tRXh4OJydnWFnZ4fRo0cjNzfXoO3ix8zNzLgrKFpZAb17AwEBQEEBDbcQQvSvQWH+ww8/wMXFRWuZWCzGzp07cezYMb02rD6OHTuG8PBwnDx5EocPH4ZCocDzzz+P4uJidZkZM2bgt99+w549e3Ds2DFkZWVh1KhRBm0X3zM3N+fC3NaWG2YZNgxwcgLkcu3yCgVQWAjcvQvcvg3cugWUlFR+KBBCSF0aNMzStm3bGtcNHDhQcGMa6uDBg1qPt27dChcXF6SkpGDw4MEoLCzE5s2bsWPHDjz33HMAgC1btsDX1xcnT57E008/bZB2aYZ5RQUX5gAgFgMeHsA//wCtWnHLioq4A6SOjtx6BwduWV4eV09ZGTdUIxIBXl6As7NBmkwIaeEMdimoK1euYOLEiUbtsRcWFgLghn8AICUlBQqFAkFBQeoyXbp0gZeXFxITEw0W5prDLAoFYGdXuc7dHTh1CrhzB7C3B7KzK4dgbG255+Tncz3zsjJuWCYriwv4e/eA3Fyu/rIy7sPCzo77IHBw4K7SqFTSl5VI3RQKQKXi7rOzufeUTMb9Z2hjw73/qrK0rDy4X5OqZezsuPeuVMq9t8vLKydvkUiqb8fGhqujsJD7e9D4JxsA914XibRPJpBIuHt+fzRJJJXbA7gO0a1bNbdfIuHKNPQQ4IMH3N9nU57kYLAwLy8vx4kTJwxVfTUqlQoREREYOHAgunfvDgDIycmBWCyGg4ODVllXV1fk5OTorKesrAxlGr99edUxkXrQPACqVFb2zAHA0xMYMoR7Q8nlgIsL0LMn92bnOTtX74GXlwPnz3Nv/ocPuT++8nKu115SwtWnVHJvQn6duTnXBv7eyor72dKSu6Kj5jozM+65SiW3TvPkJJWqcl8Y49bz5ZRKbr1YzN2rVJXL+deAr4Ov09KS+1lzGWOVbeG3URVjXBv5PzTNNmrWU3WZ5mN+PxnTvpWXc0GgGTQFBdx+8WckiURcu8zNuf2Ty7l9uH+/chsiEfeBzVhlyJSWcnXz27Kw4H6/CgXw6BH3e+FfXzMzLsBUKu75Mhm3vKKCe15xMReQKhWQkcEN30kk3OOyMq5+GxsuVCwsuHUPHnDvGTc37mc+uPnfu41N5e/I25vbv6phVlO48a8x/5rWR23l+HWtW9deztGxftuqWrdIBLRvzz3WfB9pzkNf1weWLra2lZ2xpmIyF2kNDw9Hamqq4A+QyMhILFy4UFAdmj1zoPKPGuB60M88w715cnO5P0x7+7rrFIuBfv10b+vePa73U1TE/UFbWnL3KhUX9MXFXGgolZW38vLKIOZDlb/VpK4/WP6PRfO+pnJ14T8MamtLTct1rdMMe80y/O+If8yvKy+v/FCzsKj8Y+c/MIHK32tNo48VFZUfUubm2vut2UcoLdVup0hUOUR37171dXzvz8mJCx7N8GGMC25Ly8rfgUzG3QAuJFu35trerh23X05OwLVr3GN7e64sv62qr61KVfka6fo91jfUNd9vuurjOxC66uJP+eXXaZarqKj8z5RvK3/Pf5Dzv0uFovK/2ZKSyk4J/+HWEBUV3DEvT8+GP1dfGh3m//nPf9C3b1/07t0bPXr0gFgs1me7GmTq1KnYv38/4uPjtcb13dzcUF5ejoKCAq3eeW5uLtzc3HTWNW/ePMycOVP9WC6Xw7OBvyHNMXOA63lVZWbG9eCEsrDgelwNaZtCUdl754O8vJx7oz96VNmz1gw8zQ8IKyvuzS+Vcv8OK5VcCDx4wD3X3p7rrfLDQAoF18sFuHpat+YeFxVp/3E5OnK9G7mcCyS+R8o/r7ycCx4+mADtwNEMZP6eMe5Dk/8GbuvWXB2af9wAkJNTOXxlacm1u6CA+/AVi7myfM9LIuHK8j3zoqLKHry1NXfjQ4WxyteN/1A1M+N6wKWl3LEROzvuNeUD6uFD7nXkX6uyMi7Uzcy4/ejQgWvbtWuAjw+3Pb5MeTn3frh2jXu+nx+3nQcPuOM1IhH332DVobg2ber3/tF8Xm0fqIZW01CiZsepLtbWlT9rDoU2lj7qEKLRYX7x4kVs374dxcXFsLS0RNeuXdGnTx/07dsXffr0gZkR/t9gjOGDDz5ATEwM4uLi4O3trbW+b9++sLS0RGxsLEaPHg0ASEtLw61btxAYGKizTolEAklD3hE6aPbMRaKGvcEMje8l6svjrx0A0O6h1nKsHED9w0Mf6rOthnwg8h4fmhHEy6t+5arug4cH0LVrzeVrW0dMU6PDPCEhAYwxpKWl4cyZM+pbTEwMCh53wwz9RaLw8HDs2LEDv/zyC+zt7dXj4DKZDNbW1pDJZJgwYQJmzpwJJycnSKVSfPDBBwgMDDTYwU9Ae4iA72ERQoghCRozF4lE6NKlC7p06YI333xTvfz69etISUnB2bNnBTewNhs2bAAADB06VGv5li1bMH78eADAqlWrYGZmhtGjR6OsrAzBwcH4+uuvDdouvmfOj5c24QgUIeQJIWJ0UZVayeVyyGQyFBYWQqp5ykkt+vUDkpOB998HevQA3nyzfgc5CSFPjsZkS23oitsGwPfMVSpuiIV65oQQQ6MwN4CqY+YU5oQQQ6MwNwDNMXNra+OcqkUIebJRmBsA3zPnv41HCCGGRmFuAHyYq1QU5oQQ46AwNwDNnjmdxUIIMQYKcwPQ/Aq6rq/yE0KIvlGYGwDfM5dItK//QAghhkJhbgB8z1wspjAnhBgHhbkBaPbM6RxzQogxUJgbAN8zNzfXPckCIYToG4W5AfDX57awoDAnhBgHhbkBaJ7NQvNxEkKMgcLcADRnGqIwJ4QYA4W5AWj2zGmYhRBiDBQ1elZ1FvqmnK2bNBOas0lrLmvo8obcCy1TVzmhz2lo2frWU5/n1vQcIeV4rVppz6VoRBTmeqZ5+dsWN10cY9wOqFSVn0r8fdWQaeitaj21leO3oflzfW6az9O8VV3Ol9VsU9XlmutrW6ZSca8Z/9pVfQ11vZZ8uarLq5bVtb4+z6vrNaqpTH3W1fb7qOt3VdN6zeVVy9S0Tld7GlJPY55b071mncHBwObNaAoU5nrW7MJcoeCmbi8t5aaG17wvLgZKSrjH5eXc+JBmMDDGLS8u5sorFNxjhUL3TamsvPEhV/Vx1QCsLezru6yhHxhCt6e5jBBNj+chbgoU5nrGj5cDRrwuy6NH3JuooAB4+JALXz6wy8oqw7asjGvgw4dAYSEX5MXFQFFR5bKiosrlJSXc84h+mJlxF7cXibif+cf1XV51fW3L+VvV7WouAyqfy/+s66ZZh+Zz6/s8zTK6ftYci6ypbl3bqavemsrWVb7qOs3Hmh/gVcvn5QHPPYemQmGuZ5o98/btDbQRxrjgzcsDsrKA69e5IOeDmQ9nuZy7FRRw5R884H5uaI9SJOI+mcRi7t8NiaRyPjzNe/70HXNz7o9R8zG/jF/O/8zfgJpDquofJn+KUNXy/DJdgaAr7KrWoXmrGia6gqvq83Wt1/xZ83Wv6d97zcd1red/rhoyhlTfbVWdkUXz9aitbH3X1zTjiyHW17QvutZ17667XiOgMNczzTDX23EQxrhe8oMHwL17wOnTQGoqcPMmcPcukJ8P5OZy6+vD3BxwdAScnAAHB+5nR0fuZwcHwNYWsLPjQloi4cqrVLqHUnT9Yev6g6/vH2Rdf9g1Pb+2uhqyrmoQ17VNXY91fQBU7XHXta5qPbqeq7lc1/Z17U99e7X1WVZbmZpet4a8tjUtE7peyHPqem4Tjq1SmOuZ5jBLg09LrKioDO30dODKFSAtDbh2Dbh9m+uJ37tX+9CHvT0X0k5OgLMzd6v6s0zGhUBFBTf0onkrL+fq4QO7ooJ7o1pacnVbWXE3/sIzNfXIa+tZA437I6uprK7lDQmU2u5rWlafgCTEiCjM9YzvmWt2oKrhh0lu3gROnQIuXgSuXuUCOycHuH+/8mi7LmZmgLs74OkJtGnD3bdty900pzaqqKg8YFlezoV1fj7Xmwe40JVIuJtUygW9oyPXM7ex4W6awU0hRUizRWGuZ3zPXPO/X7U7d4AffgAOHgQuX+Z62jWRSAA3N6B1a8DFhTt/tVUrrnfN96yrnhVy5472WKqFBRfC/I0Pa5lMO7BtbLjtEUJaLApzPXOWKhA9+xay8iWwvF4MpOUAiYnAL79wY92ag+oAF9SenlxQ8+PWrVtzPWX+K6QWFpWXYLSw0D4AqTnUwS/ne9v8kAh/o28wEWKyKMz1zKbkHt77sgP3QNd3Bzw8gB49gHbtuKESOzuuZ2xrywW6iwv3c9Vxaf5G1wcghOhAyaBvCgVUbh6oePAQliIFRHZ2XEB37gx07coFuI0NN4Ti4cH1xGUy7uAiXZWLENJIFOb65uUFszuZEN++zR10NDOrHB4Ribietr09TUFECNErCnNDMDMDvLyauhWEkCfIE3NEbP369WjXrh2srKwQEBCAU6dONXWTCCFEb56IMP/xxx8xc+ZMfPrppzhz5gx69uyJ4OBg5NV2aiAhhLQgT0SYr1y5EpMmTcI777yDrl27Ijo6GjY2Nvjuu++aummEEKIXJh/m5eXlSElJQVBQkHqZmZkZgoKCkJiYWK18WVkZ5HK51o0QQpo7kw/ze/fuQalUwrXKVa9cXV2Ro+Paw5GRkZDJZOqbp6ensZpKCCGNZvJh3lDz5s1DYWGh+paZmdnUTSKEkDqZ/KmJrVq1grm5OXJzc7WW5+bmws3NrVp5iUQCicZ1Stjja53QcAshRJ/4TGF6uha9yYe5WCxG3759ERsbi5EjRwIAVCoVYmNjMXXq1Dqf//DhQwCg4RZCiEE8fPgQMplMcD0mH+YAMHPmTISFhcHf3x/9+/fH6tWrUVxcjHfeeafO53p4eCAzMxP29vYQ1fMSsHK5HJ6ensjMzIRUKhXa/CZlKvtiKvsB0L40Vw3dF8YYHj58CA8PD71s/4kI89dffx13797FggULkJOTg169euHgwYPVDorqYmZmhrZt2zZqu1KptMW/QXmmsi+msh8A7Utz1ZB90UePnPdEhDkATJ06tV7DKoQQ0hLR2SyEEGICKMwNQCKR4NNPP9U6K6alMpV9MZX9AGhfmqum3hcR09d5MYQQQpoM9cwJIcQEUJgTQogJoDAnhBATQGFOCCEmgMJcz5rbjEaRkZHo168f7O3t4eLigpEjRyItLU2rTGlpKcLDw+Hs7Aw7OzuMHj262rVsbt26heHDh8PGxgYuLi6YPXs2KioqtMrExcWhT58+kEgk6NChA7Zu3WrQfYuKioJIJEJERESL3Jc7d+7grbfegrOzM6ytreHn54fk5GT1esYYFixYAHd3d1hbWyMoKAhXrlzRqiM/Px+hoaGQSqVwcHDAhAkTUFRUpFXmwoULeOaZZ2BlZQVPT08sW7ZMb/ugVCoxf/58eHt7w9raGj4+Pli8eLHW9Uaa637Ex8djxIgR8PDwgEgkwr59+7TWG7Pde/bsQZcuXWBlZQU/Pz8cOHCg4TvEiN7s2rWLicVi9t1337G///6bTZo0iTk4OLDc3Nwma1NwcDDbsmULS01NZefOnWMvvfQS8/LyYkVFReoy//nPf5inpyeLjY1lycnJ7Omnn2YDBgxQr6+oqGDdu3dnQUFB7OzZs+zAgQOsVatWbN68eeoy169fZzY2NmzmzJns0qVLbN26dczc3JwdPHjQIPt16tQp1q5dO9ajRw82ffr0Frcv+fn57KmnnmLjx49nSUlJ7Pr16+zQoUPs6tWr6jJRUVFMJpOxffv2sfPnz7NXXnmFeXt7s0ePHqnLvPDCC6xnz57s5MmT7Pjx46xDhw5s7Nix6vWFhYXM1dWVhYaGstTUVLZz505mbW3NvvnmG73sx+eff86cnZ3Z/v37WUZGBtuzZw+zs7Nja9asafb7ceDAAfbxxx+zvXv3MgAsJiZGa72x2p2QkMDMzc3ZsmXL2KVLl9gnn3zCLC0t2cWLFxu0PxTmetS/f38WHh6ufqxUKpmHhweLjIxswlZpy8vLYwDYsWPHGGOMFRQUMEtLS7Znzx51mcuXLzMALDExkTHGvenNzMxYTk6OusyGDRuYVCplZWVljDHG5syZw7p166a1rddff50FBwfrfR8ePnzIOnbsyA4fPsyGDBmiDvOWtC///e9/2aBBg2pcr1KpmJubG/vyyy/VywoKCphEImE7d+5kjDF26dIlBoCdPn1aXeb3339nIpGI3blzhzHG2Ndff80cHR3V+8Zvu3PnznrZj+HDh7N3331Xa9moUaNYaGhoi9qPqmFuzHaPGTOGDR8+XKs9AQEB7L333mvQPtAwi540dEajplJYWAgAcHJyAgCkpKRAoVBotbtLly7w8vJStzsxMRF+fn5a17IJDg6GXC7H33//rS6jWQdfxhD7Hh4ejuHDh1fbXkval19//RX+/v547bXX4OLigt69e2PTpk3q9RkZGcjJydFqh0wmQ0BAgNa+ODg4wN/fX10mKCgIZmZmSEpKUpcZPHgwxGKx1r6kpaXhwYMHgvdjwIABiI2NRXp6OgDg/PnzOHHiBF588cUWtR9VGbPd+nq/UZjrSUNnNGoKKpUKERERGDhwILp37w4AyMnJgVgshoODg1ZZzXbn5OTo3C9+XW1l5HI5Hj16pLd92LVrF86cOYPIyMhq61rSvly/fh0bNmxAx44dcejQIUyZMgXTpk3Dtm3btNpS2/spJycHLi4uWustLCzg5OTUoP0VYu7cuXjjjTfQpUsXWFpaonfv3oiIiEBoaGiL2o+qjNnumso0dL+emAttEa5Hm5qaihMnTjR1UxolMzMT06dPx+HDh2FlZdXUzRFEpVLB398fS5cuBQD07t0bqampiI6ORlhYWBO3rv52796N7du3Y8eOHejWrRvOnTuHiIgIeHh4tKj9MAXUM9eThs5oZGxTp07F/v37cfToUa1L+rq5uaG8vBwFBQVa5TXb7ebmpnO/+HW1lZFKpbC2ttbLPqSkpCAvLw99+vSBhYUFLCwscOzYMaxduxYWFhZwdXVtMfvi7u6Orl27ai3z9fXFrVu3tNpS2/vJzc0NeXl5WusrKiqQn5/foP0VYvbs2ereuZ+fH95++23MmDFD/Z9TS9mPqozZ7prKNHS/KMz1RHNGIx4/o1FgYGCTtYsxhqlTpyImJgZHjhyBt7e31vq+ffvC0tJSq91paWm4deuWut2BgYG4ePGi1hv38OHDkEql6kAKDAzUqoMvo899HzZsGC5evIhz586pb/7+/ggNDVX/3FL2ZeDAgdVOEU1PT8dTTz0FAPD29oabm5tWO+RyOZKSkrT2paCgACkpKeoyR44cgUqlQkBAgLpMfHw8FAqF1r507twZjo6OgvejpKQEZmbaMWJubg6VStWi9qMqY7Zbb++3Bh0uJbXatWsXk0gkbOvWrezSpUts8uTJzMHBQevMCWObMmUKk8lkLC4ujmVnZ6tvJSUl6jL/+c9/mJeXFzty5AhLTk5mgYGBLDAwUL2eP53v+eefZ+fOnWMHDx5krVu31nk63+zZs9nly5fZ+vXrDXpqIk/zbJaWtC+nTp1iFhYW7PPPP2dXrlxh27dvZzY2Nux///ufukxUVBRzcHBgv/zyC7tw4QILCQnReWpc7969WVJSEjtx4gTr2LGj1qlxBQUFzNXVlb399tssNTWV7dq1i9nY2Ojt1MSwsDDWpk0b9amJe/fuZa1atWJz5sxp9vvx8OFDdvbsWXb27FkGgK1cuZKdPXuW3bx506jtTkhIYBYWFmz58uXs8uXL7NNPP6VTE5uDdevWMS8vLyYWi1n//v3ZyZMnm7Q9AHTetmzZoi7z6NEj9v777zNHR0dmY2PDXn31VZadna1Vz40bN9iLL77IrK2tWatWrdiHH37IFAqFVpmjR4+yXr16MbFYzNq3b6+1DUOpGuYtaV9+++031r17dyaRSFiXLl3Yxo0btdarVCo2f/585urqyiQSCRs2bBhLS0vTKnP//n02duxYZmdnx6RSKXvnnXfYw4cPtcqcP3+eDRo0iEkkEtamTRsWFRWlt32Qy+Vs+vTpzMvLi1lZWbH27duzjz/+WOtUvOa6H0ePHtX5txEWFmb0du/evZt16tSJicVi1q1bN/Z///d/Dd4fugQuIYSYABozJ4QQE0BhTgghJoDCnBBCTACFOSGEmAAKc0IIMQEU5oQQYgIozAkhxARQmBMC7mvpo0ePhlQqhUgkqnZ9FwD47LPP0KtXL6O3rS5Dhw7Vmm2JPJkozEmTGD9+PEQiEaKiorSW79u3DyKRyOjt2bZtG44fP46//voL2dnZkMlk1crMmjVL6xoa48ePx8iRI43Wxri4OJ0fNHv37sXixYuN1g7SPFGYkyZjZWWFL774wiCTCzTUtWvX4Ovri+7du8PNzU3nB4qdnR2cnZ31vu3y8nJBz3dycoK9vb2eWkNaKgpz0mSCgoLg5uamc6IJTT///DO6desGiUSCdu3aYcWKFQ3eVm11DB06FCtWrEB8fDxEIhGGDh2qsw7NYZbPPvsM27Ztwy+//AKRSASRSIS4uDgA3HXXx4wZAwcHBzg5OSEkJAQ3btxQ18P36D///HN4eHigc+fOAIAffvgB/v7+sLe3h5ubG95880311R1v3LiBZ599FgDg6OgIkUiE8ePHq9uvOczy4MEDjBs3Do6OjrCxscGLL76oNRHx1q1b4eDggEOHDsHX1xd2dnZ44YUXkJ2drS4TFxeH/v37w9bWFg4ODhg4cCBu3rzZ4NedGA+FOWky5ubmWLp0KdatW4fbt2/rLJOSkoIxY8bgjTfewMWLF/HZZ59h/vz52Lp1a723U1cde/fuxaRJkxAYGIjs7Gzs3bu3zjpnzZqFMWPGqEMwOzsbAwYMgEKhQHBwMOzt7XH8+HEkJCSow1KzBx4bG4u0tDQcPnwY+/fvBwAoFAosXrwY58+fx759+3Djxg11YHt6euLnn38GwF3WNzs7G2vWrNHZtvHjxyM5ORm//vorEhMTwRjDSy+9pHUZ1pKSEixfvhw//PAD4uPjcevWLcyaNQsAd03ukSNHYsiQIbhw4QISExMxefLkJhn+Ig3Q4EtzEaIHYWFhLCQkhDHG2NNPP62eFDgmJoZpvi3ffPNN9q9//UvrubNnz2Zdu3at97bqU8f06dPZkCFDaq3n008/ZT179tS5D7wffviBde7cmalUKvWysrIyZm1tzQ4dOqR+nqurq9aVBXU5ffo0A6C+Ch9/lb8HDx5oldO8cmR6ejoDwBISEtTr7927x6ytrdnu3bsZY4xt2bKFAWBXr15Vl1m/fj1zdXVljHFXAgTA4uLiam0faV6oZ06a3BdffIFt27bh8uXL1dZdvnwZAwcO1Fo2cOBAXLlyBUqlsl7166OO+jp//jyuXr0Ke3t72NnZwc7ODk5OTigtLcW1a9fU5fz8/LQm+QW4/yBGjBgBLy8v2NvbY8iQIQCgnn2oPi5fvgwLCwv15AgA4OzsjM6dO2u9vjY2NvDx8VE/dnd3Vw/pODk5Yfz48QgODsaIESOwZs0arSEY0jxRmJMmN3jwYAQHB2PevHlN3RTBioqK0LdvX63ZkM6dO4f09HS8+eab6nK2trZazysuLkZwcDCkUim2b9+O06dPIyYmBoDwA6S6WFpaaj0WiURgGlfD3rJlCxITEzFgwAD8+OOP6NSpE06ePKn3dhD9oQmdSbMQFRWFXr16qQ8G8nx9fZGQkKC1LCEhAZ06dYK5uXm96tZHHbqIxeJqPfs+ffrgxx9/hIuLC6RSab3r+ueff3D//n1ERUXB09MTAJCcnFxtewBq/W/C19cXFRUVSEpKwoABAwAA9+/fR1paWrU5R+vSu3dv9O7dG/PmzUNgYCB27NiBp59+ukF1EOOhnjlpFvz8/BAaGoq1a9dqLf/www8RGxuLxYsXIz09Hdu2bcNXX32lPlgHcHODfvXVVzXWXZ86GqNdu3a4cOEC0tLScO/ePSgUCoSGhqJVq1YICQnB8ePHkZGRgbi4OEybNq3Gg7wA4OXlBbFYjHXr1uH69ev49ddfq507/tRTT0EkEmH//v24e/cuioqKqtXTsWNHhISEYNKkSThx4gTOnz+Pt956C23atEFISEi99isjIwPz5s1DYmIibt68iT/++ANXrlyBr69vw14gYlQU5qTZWLRokXoiYF6fPn2we/du7Nq1C927d8eCBQuwaNEi9VkeAHeO+L1792qstz51NMakSZPQuXNn+Pv7o3Xr1khISICNjQ3i4+Ph5eWFUaNGwdfXFxMmTEBpaWmtPfXWrVtj69at2LNnD7p27YqoqCgsX75cq0ybNm2wcOFCzJ07F66urpg6darOurZs2YK+ffvi5ZdfRmBgIBhjOHDgQLWhlZrY2Njgn3/+wejRo9GpUydMnjwZ4eHheO+99+r/4hCjo2njCCHEBFDPnBBCTACFOSGEmAAKc0IIMQEU5oQQYgIozAkhxARQmBNCiAmgMCeEEBNAYU4IISaAwpwQQkwAhTkhhJgACnNCCDEBFOaEEGIC/h9viYtYSgMLJAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 360x270 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "norm_mean_32 = np.mean(norm_32, axis = 0)\n",
    "norm_mean_16 = np.mean(norm_16, axis = 0)\n",
    "norm_stdev_32 = np.std(norm_32, axis = 0)\n",
    "norm_stdev_16 = np.std(norm_16, axis = 0)\n",
    "\n",
    "x = [i for i in range(len(norm_mean_32))]\n",
    "#plt.figure(figsize=(4.8,3.6))\n",
    "plt.figure(figsize=(3.6,2.7))\n",
    "plt.xlabel('No. of iterations')\n",
    "plt.ylabel('$L^2$ norm of weights')\n",
    "plt.plot(x, norm_mean_32, label = \"Float32\", color = 'red')\n",
    "plt.plot(x, norm_mean_16, label = \"Float16\", color = 'blue')\n",
    "print((norm_mean_32 - norm_stdev_32).shape)\n",
    "print(np.full(10000, 0).shape)\n",
    "plt.fill_between(x, np.maximum(norm_mean_32 - norm_stdev_32, np.full(10000, 0)), np.minimum(norm_mean_32 + norm_stdev_32, np.full(10000, 100)), color = 'red', alpha = 0.3)\n",
    "plt.fill_between(x, np.maximum(norm_mean_16 - norm_stdev_16, np.full(10000, 0)), np.minimum(norm_mean_16 + norm_stdev_16, np.full(10000, 100)), color = 'blue', alpha = 0.3)\n",
    "leg1 = plt.legend(loc = 'right', frameon=False)\n",
    "plt.savefig(\"norm-weights.pdf\", bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"float32-norm-weights\", norm_32)\n",
    "np.save(\"float16-norm-weights\", norm_16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "3v3fNbs6m91w"
   },
   "outputs": [],
   "source": [
    "np.save(\"float32-constant-weights\", const_32)\n",
    "np.save(\"float16-constant-weights\", const_16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "const_32 = np.load(\"./float32-constant-weights.npy\")\n",
    "const_16 = np.load(\"./float16-constant-weights.npy\")\n",
    "norm_32 = np.load(\"./float32-norm-weights.npy\")\n",
    "norm_16 = np.load(\"./float16-norm-weights.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Set the default float type to float16\n",
      "Compiling model...\n",
      "'compile' took 0.013237 s\n",
      "\n",
      "0.0939 -0.3499\n",
      "there was no change, iteration 497, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0031280517578125 lr * grad = 3.1280517578125002e-06\n",
      "there was no change, iteration 498, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.003108978271484375 lr * grad = 3.1089782714843752e-06\n",
      "there was no change, iteration 499, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0030765533447265625 lr * grad = 3.0765533447265625e-06\n",
      "there was no change, iteration 500, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0030956268310546875 lr * grad = 3.0956268310546875e-06\n",
      "there was no change, iteration 501, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.003173828125 lr * grad = 3.1738281250000003e-06\n",
      "there was no change, iteration 502, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0031566619873046875 lr * grad = 3.1566619873046876e-06\n",
      "there was no change, iteration 503, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.003154754638671875 lr * grad = 3.1547546386718753e-06\n",
      "there was no change, iteration 504, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0031337738037109375 lr * grad = 3.1337738037109375e-06\n",
      "there was no change, iteration 505, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.003078460693359375 lr * grad = 3.078460693359375e-06\n",
      "there was no change, iteration 506, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.00302886962890625 lr * grad = 3.02886962890625e-06\n",
      "there was no change, iteration 507, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.00299835205078125 lr * grad = 2.99835205078125e-06\n",
      "there was no change, iteration 508, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029926300048828125 lr * grad = 2.9926300048828124e-06\n",
      "there was no change, iteration 509, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.003055572509765625 lr * grad = 3.055572509765625e-06\n",
      "there was no change, iteration 510, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029888153076171875 lr * grad = 2.9888153076171874e-06\n",
      "there was no change, iteration 511, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.00301361083984375 lr * grad = 3.01361083984375e-06\n",
      "there was no change, iteration 512, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.003009796142578125 lr * grad = 3.009796142578125e-06\n",
      "there was no change, iteration 513, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029163360595703125 lr * grad = 2.9163360595703128e-06\n",
      "there was no change, iteration 514, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.00292205810546875 lr * grad = 2.92205810546875e-06\n",
      "there was no change, iteration 515, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.00295257568359375 lr * grad = 2.95257568359375e-06\n",
      "there was no change, iteration 516, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029201507568359375 lr * grad = 2.9201507568359378e-06\n",
      "there was no change, iteration 517, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.00284576416015625 lr * grad = 2.84576416015625e-06\n",
      "there was no change, iteration 518, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029048919677734375 lr * grad = 2.9048919677734378e-06\n",
      "there was no change, iteration 519, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002857208251953125 lr * grad = 2.857208251953125e-06\n",
      "there was no change, iteration 520, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0028591156005859375 lr * grad = 2.8591156005859377e-06\n",
      "there was no change, iteration 521, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029315948486328125 lr * grad = 2.9315948486328124e-06\n",
      "there was no change, iteration 522, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029449462890625 lr * grad = 2.9449462890625e-06\n",
      "there was no change, iteration 523, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002857208251953125 lr * grad = 2.857208251953125e-06\n",
      "there was no change, iteration 524, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002887725830078125 lr * grad = 2.887725830078125e-06\n",
      "there was no change, iteration 525, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029354095458984375 lr * grad = 2.9354095458984374e-06\n",
      "there was no change, iteration 526, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002948760986328125 lr * grad = 2.948760986328125e-06\n",
      "there was no change, iteration 527, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.00293731689453125 lr * grad = 2.93731689453125e-06\n",
      "there was no change, iteration 528, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002902984619140625 lr * grad = 2.902984619140625e-06\n",
      "there was no change, iteration 529, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002910614013671875 lr * grad = 2.910614013671875e-06\n",
      "there was no change, iteration 530, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.00292205810546875 lr * grad = 2.92205810546875e-06\n",
      "there was no change, iteration 531, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002902984619140625 lr * grad = 2.902984619140625e-06\n",
      "there was no change, iteration 532, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.00290679931640625 lr * grad = 2.90679931640625e-06\n",
      "there was no change, iteration 533, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002971649169921875 lr * grad = 2.971649169921875e-06\n",
      "there was no change, iteration 534, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029735565185546875 lr * grad = 2.9735565185546874e-06\n",
      "there was no change, iteration 535, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002956390380859375 lr * grad = 2.956390380859375e-06\n",
      "there was no change, iteration 536, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029449462890625 lr * grad = 2.9449462890625e-06\n",
      "there was no change, iteration 537, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.00299072265625 lr * grad = 2.99072265625e-06\n",
      "there was no change, iteration 538, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.00299072265625 lr * grad = 2.99072265625e-06\n",
      "there was no change, iteration 539, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002941131591796875 lr * grad = 2.941131591796875e-06\n",
      "there was no change, iteration 540, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029449462890625 lr * grad = 2.9449462890625e-06\n",
      "there was no change, iteration 541, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0030364990234375 lr * grad = 3.0364990234375e-06\n",
      "there was no change, iteration 542, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.003009796142578125 lr * grad = 3.009796142578125e-06\n",
      "there was no change, iteration 543, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.003002166748046875 lr * grad = 3.002166748046875e-06\n",
      "there was no change, iteration 544, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.003017425537109375 lr * grad = 3.017425537109375e-06\n",
      "there was no change, iteration 545, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029754638671875 lr * grad = 2.9754638671875e-06\n",
      "there was no change, iteration 546, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002910614013671875 lr * grad = 2.910614013671875e-06\n",
      "there was no change, iteration 547, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002895355224609375 lr * grad = 2.895355224609375e-06\n",
      "there was no change, iteration 548, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0028839111328125 lr * grad = 2.8839111328125e-06\n",
      "there was no change, iteration 549, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002925872802734375 lr * grad = 2.925872802734375e-06\n",
      "there was no change, iteration 550, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.00293731689453125 lr * grad = 2.93731689453125e-06\n",
      "there was no change, iteration 551, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029144287109375 lr * grad = 2.9144287109375e-06\n",
      "there was no change, iteration 552, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029659271240234375 lr * grad = 2.9659271240234374e-06\n",
      "there was no change, iteration 553, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002941131591796875 lr * grad = 2.941131591796875e-06\n",
      "there was no change, iteration 554, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002956390380859375 lr * grad = 2.956390380859375e-06\n",
      "there was no change, iteration 555, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029277801513671875 lr * grad = 2.9277801513671878e-06\n",
      "there was no change, iteration 556, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002880096435546875 lr * grad = 2.880096435546875e-06\n",
      "there was no change, iteration 557, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029277801513671875 lr * grad = 2.9277801513671878e-06\n",
      "there was no change, iteration 558, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.00292205810546875 lr * grad = 2.92205810546875e-06\n",
      "there was no change, iteration 559, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0028934478759765625 lr * grad = 2.8934478759765627e-06\n",
      "there was no change, iteration 560, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.00286102294921875 lr * grad = 2.86102294921875e-06\n",
      "there was no change, iteration 561, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029201507568359375 lr * grad = 2.9201507568359378e-06\n",
      "there was no change, iteration 562, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029010772705078125 lr * grad = 2.9010772705078127e-06\n",
      "there was no change, iteration 563, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.00289154052734375 lr * grad = 2.89154052734375e-06\n",
      "there was no change, iteration 564, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029087066650390625 lr * grad = 2.9087066650390628e-06\n",
      "there was no change, iteration 565, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002899169921875 lr * grad = 2.899169921875e-06\n",
      "there was no change, iteration 566, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002918243408203125 lr * grad = 2.918243408203125e-06\n",
      "there was no change, iteration 567, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002899169921875 lr * grad = 2.899169921875e-06\n",
      "there was no change, iteration 568, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.002864837646484375 lr * grad = 2.864837646484375e-06\n",
      "there was no change, iteration 569, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0028781890869140625 lr * grad = 2.8781890869140627e-06\n",
      "there was no change, iteration 570, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.00287628173828125 lr * grad = 2.87628173828125e-06\n",
      "there was no change, iteration 571, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.00290679931640625 lr * grad = 2.90679931640625e-06\n",
      "there was no change, iteration 572, prev weight was -0.5, prev_weight + lr * grad = -0.5 grad 0.0029659271240234375 lr * grad = 2.9659271240234374e-06\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[36], line 36\u001b[0m\n\u001b[1;32m     32\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(iterations):\n\u001b[1;32m     33\u001b[0m     model\u001b[38;5;241m.\u001b[39mtrain_state\u001b[38;5;241m.\u001b[39mset_data_train(\n\u001b[1;32m     34\u001b[0m         \u001b[38;5;241m*\u001b[39mmodel\u001b[38;5;241m.\u001b[39mdata\u001b[38;5;241m.\u001b[39mtrain_next_batch(model\u001b[38;5;241m.\u001b[39mbatch_size)\n\u001b[1;32m     35\u001b[0m     )\n\u001b[0;32m---> 36\u001b[0m     \u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain_step\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     37\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain_state\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mX_train\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     38\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain_state\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43my_train\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     39\u001b[0m \u001b[43m        \u001b[49m\u001b[43mmodel\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain_state\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mtrain_aux_vars\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     40\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     42\u001b[0m     X_test, y_test \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mdata\u001b[38;5;241m.\u001b[39mtest()\n\u001b[1;32m     43\u001b[0m     y_pred \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mpredict(X_test)\n",
      "File \u001b[0;32m~/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages/tensorflow/python/util/traceback_utils.py:150\u001b[0m, in \u001b[0;36mfilter_traceback.<locals>.error_handler\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    148\u001b[0m filtered_tb \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    149\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 150\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mfn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    151\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    152\u001b[0m   filtered_tb \u001b[38;5;241m=\u001b[39m _process_traceback_frames(e\u001b[38;5;241m.\u001b[39m__traceback__)\n",
      "File \u001b[0;32m~/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py:830\u001b[0m, in \u001b[0;36mFunction.__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    826\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m trace\u001b[38;5;241m.\u001b[39mTrace(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_name) \u001b[38;5;28;01mas\u001b[39;00m tm:\n\u001b[1;32m    827\u001b[0m   \u001b[38;5;66;03m# TODO(cheshire): Do not duplicate the XLAControlFlowContext annotation.\u001b[39;00m\n\u001b[1;32m    828\u001b[0m   compiler \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mxla\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_jit_compile \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnonXla\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m--> 830\u001b[0m \u001b[43m  \u001b[49m\u001b[38;5;28;43;01mwith\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mOptionalXlaContext\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_jit_compile\u001b[49m\u001b[43m)\u001b[49m\u001b[43m:\u001b[49m\n\u001b[1;32m    831\u001b[0m \u001b[43m    \u001b[49m\u001b[43mresult\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwds\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    833\u001b[0m   new_tracing_count \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mexperimental_get_tracing_count()\n",
      "File \u001b[0;32m~/Documents/mixed-precision-sciml/venv/lib/python3.11/site-packages/tensorflow/python/eager/polymorphic_function/polymorphic_function.py:446\u001b[0m, in \u001b[0;36mOptionalXlaContext.__exit__\u001b[0;34m(self, t, value, traceback)\u001b[0m\n\u001b[1;32m    443\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mxla_context:\n\u001b[1;32m    444\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mxla_context\u001b[38;5;241m.\u001b[39mEnter()\n\u001b[0;32m--> 446\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__exit__\u001b[39m(\u001b[38;5;28mself\u001b[39m, t, value, traceback):\n\u001b[1;32m    447\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mxla_context:\n\u001b[1;32m    448\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mxla_context\u001b[38;5;241m.\u001b[39mExit()\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# the two times when we want to compare gradient and weight\n",
    "TRAINING_I = 2\n",
    "STAGNANT_I = 5_000\n",
    "\n",
    "seed = 0\n",
    "def get_gradients_of_weights(model16):\n",
    "    gradients161d = np.concatenate([layer.numpy().ravel() for layer in gradients16])\n",
    "    return gradients161d\n",
    "dde.config.set_default_float(\"float16\")\n",
    "dde.config.set_random_seed(seed)\n",
    "\n",
    "geom = dde.geometry.Interval(-1, 1)\n",
    "num_train = 16\n",
    "num_test = 100\n",
    "lr = 1e-3\n",
    "data = dde.data.Function(geom, func, num_train, num_test)\n",
    "\n",
    "activation = \"tanh\"\n",
    "initializer = \"Glorot uniform\"\n",
    "net = dde.nn.FNN([1] + [10] * 2 + [1], activation, kernel_initializer =  tf.keras.initializers.glorot_uniform(seed=seed))\n",
    "\n",
    "model = dde.Model(data, net)\n",
    "model.compile(\"adam\", lr=lr, metrics=[\"l2 relative error\"])\n",
    "\n",
    "a = 0\n",
    "b = 0\n",
    "\n",
    "# also get individual weight values to do an in-depth analysis\n",
    "iterations = 10_000\n",
    "prev_weight = 0.0\n",
    "prev_grad = 0.0\n",
    "for i in range(iterations):\n",
    "    model.train_state.set_data_train(\n",
    "        *model.data.train_next_batch(model.batch_size)\n",
    "    )\n",
    "    model.train_step(\n",
    "        model.train_state.X_train,\n",
    "        model.train_state.y_train,\n",
    "        model.train_state.train_aux_vars,\n",
    "    )\n",
    "\n",
    "    X_test, y_test = model.data.test()\n",
    "    y_pred = model.predict(X_test)\n",
    "    l2r = np.linalg.norm(y_pred - y_test) / np.linalg.norm(y_test)\n",
    "\n",
    "    x_train = model.data.train_x\n",
    "    y_train = model.data.train_y\n",
    "    with tf.GradientTape() as tape:\n",
    "        y_pred = model.net.call(x_train)\n",
    "        loss_fn = dde.losses.get(\"MSE\")\n",
    "        loss = loss_fn(y_train, y_pred)\n",
    "    gradients = tape.gradient(loss,model.net.trainable_weights)\n",
    "    assert(gradients[2].shape == (10, 10)) # these are the gradients of the weights from hidden layer 1 -> 2\n",
    "    layer1neuron1tolayer2neuron1_grad = gradients[2][a][b]\n",
    "    assert(model.net.denses[1].get_weights()[0].shape == (10,10))\n",
    "    layer1neuron1tolayer2neuron1_weight = model.net.denses[1].get_weights()[0][a][b]\n",
    "    if (prev_weight == layer1neuron1tolayer2neuron1_weight):\n",
    "        # this is not totally what is going on since adam is not SGD where weight_t = weight_{t-1} + lr * grad, but it is close enough for our purposes\n",
    "        print(f\"there was no change, iteration {i}, prev weight was {prev_weight}, prev_weight + lr * grad = {(prev_weight + lr * layer1neuron1tolayer2neuron1_grad).numpy()}\", f\"grad {layer1neuron1tolayer2neuron1_grad.numpy()}\", f\"lr * grad = {lr * layer1neuron1tolayer2neuron1_grad.numpy()}\",)\n",
    "    prev_grad = layer1neuron1tolayer2neuron1_grad\n",
    "    prev_weight = layer1neuron1tolayer2neuron1_weight\n",
    "    # print(type(prev_grad.numpy()), type(prev_weight)) # both show float16\n",
    "    if i == TRAINING_I or i == STAGNANT_I:\n",
    "        print(layer1neuron1tolayer2neuron1_grad.numpy(), layer1neuron1tolayer2neuron1_weight)\n",
    "\n",
    "    model.train_state.epoch += 1\n",
    "    model.train_state.step += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}