LRdecay_BSincrease.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "EFRmSlV847CF"
   },
   "source": [
    "## MNIST "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DPRhJ_m34-PB"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torchvision.datasets as dset\n",
    "import torchvision.transforms as transforms\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from matplotlib.pyplot import figure\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mBis_QFoHJ0S"
   },
   "outputs": [],
   "source": [
    "root = './data'\n",
    "if not os.path.exists(root):\n",
    "    os.mkdir(root)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8b7iFmMoHOsy"
   },
   "outputs": [],
   "source": [
    "train_set = dset.MNIST(root=root, train=True, transform=transforms.ToTensor(), download=True)\n",
    "test_set = dset.MNIST(root=root, train=False, transform=transforms.ToTensor(), download=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mrMhrECTHkky"
   },
   "outputs": [],
   "source": [
    "batch_size = 50\n",
    "total_epoch = 30\n",
    "learning_rate = 0.05\n",
    "use_cuda = torch.cuda.is_available() # [Question 1] What is 'torch.cuda.is_available' used for?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qEPSA3LuPazn",
    "tags": [
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend"
    ]
   },
   "outputs": [],
   "source": [
    "train_loader = torch.utils.data.DataLoader(  # [Question 2] What is 'torch.utils.data.DataLoader' used for?\n",
    "                 dataset=train_set,\n",
    "                 batch_size=batch_size,\n",
    "                 shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "                dataset=test_set,\n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "class p1CNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(p1CNN, self).__init__()\n",
    "        self.c1 = nn.Conv2d(1, 6, kernel_size=3, padding=1)\n",
    "        self.c2 = nn.Conv2d(6, 1, kernel_size=3, padding=1)\n",
    "        self.mp = nn.MaxPool2d(kernel_size=2)\n",
    "        self.li = nn.Linear(7*7, 10)\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.mp(F.relu(self.c1(x)))\n",
    "        x = self.mp(F.relu(self.c2(x)))\n",
    "        x = x.view(-1, 7*7)\n",
    "        x = self.li(x)\n",
    "        return x\n",
    "\n",
    "def weights_init(m):\n",
    "    classname = m.__class__.__name__\n",
    "    for p in model.parameters():\n",
    "        p.data.fill_(1)\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "GlyL40rSPxDO"
   },
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "<All keys matched successfully>"
     },
     "metadata": {},
     "execution_count": 136
    }
   ],
   "source": [
    "model = p1CNN()\n",
    "\n",
    "if use_cuda:\n",
    "    model = model.cuda()\n",
    "\n",
    "tmp = p1CNN()\n",
    "tmp.load_state_dict(model.state_dict())\n",
    "# model.apply(weights_init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LHB6pCrAQJes"
   },
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=learning_rate) # [Question 5] What is 'optim.SGD' used for?\n",
    "criterion = nn.CrossEntropyLoss() # [Question 6] What is 'nn.CrossEntropyLoss' used for?\n",
    "lr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.5)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "OrderedDict([('c1.weight', tensor([[[[ 0.0925, -0.0441,  0.0553],\n          [-0.1138, -0.2961,  0.0047],\n          [ 0.2043, -0.0859, -0.1000]]],\n\n\n        [[[ 0.2786, -0.2202, -0.1703],\n          [ 0.1581, -0.1463,  0.0819],\n          [-0.0489,  0.0269,  0.2253]]],\n\n\n        [[[ 0.0630, -0.2880, -0.3312],\n          [ 0.0864,  0.3215, -0.1820],\n          [-0.2329,  0.1246, -0.2132]]],\n\n\n        [[[-0.1638, -0.0027,  0.2143],\n          [ 0.1054,  0.2291, -0.0825],\n          [-0.0524, -0.0930, -0.1907]]],\n\n\n        [[[-0.0395, -0.1994,  0.0414],\n          [ 0.1200,  0.1663,  0.1609],\n          [-0.0673, -0.0838, -0.1943]]],\n\n\n        [[[ 0.2045,  0.0437, -0.2027],\n          [-0.3034, -0.2230, -0.2194],\n          [-0.1230,  0.1417,  0.2724]]]], device='cuda:0')), ('c1.bias', tensor([ 0.0712, -0.1130, -0.0558,  0.1180, -0.1398,  0.2255], device='cuda:0')), ('c2.weight', tensor([[[[-0.0188,  0.0184,  0.0924],\n          [ 0.0904, -0.0386, -0.0970],\n          [-0.1123, -0.0399, -0.1308]],\n\n         [[ 0.0316, -0.0420, -0.0796],\n          [-0.0241,  0.1242, -0.1160],\n          [ 0.1200, -0.0723, -0.0770]],\n\n         [[ 0.0819,  0.1143, -0.0121],\n          [ 0.0983, -0.0685, -0.1324],\n          [ 0.1236, -0.0944,  0.0295]],\n\n         [[ 0.0060,  0.0954, -0.0018],\n          [-0.1239,  0.1180, -0.0535],\n          [-0.0246, -0.0424, -0.0836]],\n\n         [[ 0.0952, -0.0344, -0.0372],\n          [ 0.0837, -0.0964, -0.0016],\n          [-0.0924, -0.0837,  0.0697]],\n\n         [[-0.0296, -0.0500,  0.0390],\n          [-0.0352, -0.1125,  0.0396],\n          [ 0.1076, -0.0536, -0.1119]]]], device='cuda:0')), ('c2.bias', tensor([-0.0129], device='cuda:0')), ('li.weight', tensor([[-0.0918, -0.0911,  0.0627, -0.1095, -0.0411,  0.0125, -0.0689,  0.0489,\n          0.0844,  0.1316, -0.1375, -0.0384, -0.1275,  0.0487, -0.1031, -0.1107,\n          0.1413,  0.0660, -0.0525, -0.0849, -0.0154,  0.0113, -0.1389, -0.0462,\n         -0.0896, -0.0157, -0.0989, -0.0819,  0.0680,  0.0535,  0.0251, -0.0148,\n          0.0872,  0.0552,  0.0337,  0.1364, -0.0334,  0.0922, -0.1169,  0.0665,\n          0.0224, -0.0623,  0.0546, -0.0758,  0.0277,  0.1098, -0.1174,  0.0328,\n         -0.0862],\n        [ 0.1056, -0.0422, -0.0716, -0.0061, -0.0806,  0.0621, -0.0376, -0.0857,\n          0.0902,  0.1257,  0.0466,  0.0312, -0.0220, -0.1240, -0.0361,  0.0486,\n         -0.0429,  0.0917,  0.0370, -0.0669, -0.0999, -0.0324,  0.1211,  0.0580,\n         -0.0079, -0.0259, -0.0652, -0.0719, -0.1317, -0.0434, -0.0519, -0.1402,\n         -0.0181,  0.0470,  0.0680, -0.0656, -0.0819, -0.0853, -0.1316,  0.1056,\n          0.0921, -0.1254,  0.1183,  0.1205,  0.0069,  0.0105, -0.1418, -0.0630,\n          0.1178],\n        [-0.0297, -0.0807,  0.0214, -0.1154, -0.0922, -0.1406,  0.0668,  0.0093,\n         -0.0487,  0.1307, -0.1133, -0.0962, -0.1415, -0.0291,  0.1244, -0.0158,\n         -0.1413,  0.1356, -0.0251,  0.0954,  0.0614,  0.1231, -0.0789,  0.0264,\n         -0.1039,  0.0286, -0.1143, -0.0907,  0.0016,  0.0406,  0.0052,  0.0870,\n          0.0958, -0.0596, -0.1050,  0.1328, -0.0062, -0.0857, -0.0343, -0.0729,\n          0.1339, -0.0710,  0.1018,  0.0026,  0.0579, -0.0894, -0.1315, -0.0686,\n          0.0887],\n        [ 0.0497,  0.1208,  0.1305, -0.1141,  0.1317, -0.1109, -0.1208,  0.1110,\n          0.0817, -0.0444, -0.0257,  0.0680, -0.0155,  0.0483,  0.0326,  0.0303,\n          0.0438,  0.0950, -0.1104, -0.0540, -0.0772, -0.0387,  0.0580, -0.1217,\n         -0.1185,  0.1358,  0.0870,  0.0099,  0.0388, -0.1085,  0.0493, -0.1011,\n         -0.0473,  0.0136, -0.1325,  0.0466,  0.0471, -0.0962, -0.0534,  0.0037,\n         -0.0802, -0.0656,  0.0817, -0.1179, -0.1341, -0.1183, -0.0977,  0.1011,\n          0.0430],\n        [-0.0243,  0.1397, -0.0901, -0.0640,  0.0024, -0.0176, -0.0539, -0.0644,\n          0.1269, -0.0066,  0.1158,  0.0521, -0.0554, -0.1268, -0.1125, -0.0598,\n         -0.1045,  0.0522, -0.0263, -0.1391, -0.1275,  0.1308, -0.1157, -0.0838,\n          0.0871, -0.1406, -0.1259,  0.0329, -0.0954, -0.0035,  0.1343, -0.1014,\n          0.0544, -0.0562,  0.0638, -0.0263, -0.0274,  0.1227, -0.0085, -0.0898,\n          0.0940,  0.0837, -0.0630, -0.1145, -0.1253,  0.1056,  0.1158, -0.0849,\n          0.0405],\n        [ 0.0366,  0.1240,  0.0844,  0.0808,  0.0178, -0.1103, -0.0482, -0.1045,\n         -0.1165, -0.0816,  0.0781,  0.0821, -0.0779,  0.0014,  0.0112,  0.0809,\n         -0.0176,  0.0382, -0.0111,  0.0589, -0.0825, -0.0175, -0.0004, -0.0024,\n          0.0436, -0.0174, -0.0201, -0.0032,  0.0534,  0.0923, -0.0357,  0.0731,\n          0.1065,  0.0219, -0.1300,  0.0959, -0.0436,  0.0521, -0.0586, -0.0101,\n         -0.0521,  0.0044, -0.0743,  0.0626, -0.0845, -0.1183, -0.0870, -0.0849,\n          0.1377],\n        [-0.0125, -0.0649, -0.0181,  0.0489, -0.0386, -0.0888, -0.0252, -0.0178,\n         -0.0183, -0.0915,  0.1263, -0.0701, -0.0959, -0.1202, -0.0185,  0.0549,\n          0.0119,  0.1057,  0.0744, -0.1341,  0.0408,  0.0816,  0.1331,  0.0009,\n         -0.1185,  0.1097, -0.0398,  0.0713,  0.1231, -0.0274,  0.1267, -0.0767,\n         -0.0304,  0.0934, -0.0603,  0.0609,  0.0317, -0.1182,  0.1070, -0.0883,\n         -0.0988,  0.0210,  0.0589, -0.1128,  0.1027, -0.0612,  0.0469,  0.0981,\n          0.1404],\n        [-0.0425,  0.0783, -0.1148, -0.0602, -0.0022, -0.0162,  0.0640, -0.0138,\n          0.0497,  0.0945, -0.0031,  0.1175,  0.0429, -0.0838, -0.1244,  0.1280,\n         -0.0002,  0.0941, -0.0543,  0.0855,  0.0261,  0.0084,  0.0065,  0.1019,\n         -0.0629, -0.1304,  0.0176, -0.0304,  0.0995,  0.0305, -0.0760,  0.0178,\n          0.0807, -0.1141,  0.1248,  0.1381, -0.0373, -0.0938,  0.1028,  0.0882,\n          0.1189, -0.1296, -0.0793, -0.0213,  0.0047, -0.0294, -0.0448, -0.1091,\n          0.0004],\n        [-0.0536, -0.0434, -0.0177, -0.0271, -0.1135, -0.0527, -0.0058, -0.0928,\n          0.1390,  0.0758, -0.0601,  0.1165, -0.1287,  0.0847, -0.0277,  0.1254,\n         -0.0569,  0.0889,  0.0913, -0.0753, -0.1329, -0.0126, -0.0957, -0.0768,\n          0.0190,  0.1283, -0.1204,  0.0294,  0.0788,  0.0769,  0.0933,  0.1232,\n         -0.0074, -0.0427, -0.0147, -0.0651,  0.0741,  0.1151, -0.0205,  0.0272,\n          0.0411,  0.0257,  0.1428, -0.1274,  0.0011, -0.0707, -0.0702, -0.0953,\n         -0.0754],\n        [ 0.0536, -0.0307,  0.0124, -0.1100, -0.0294,  0.0544, -0.1414,  0.0229,\n         -0.0982,  0.0929,  0.0510, -0.0535, -0.0489,  0.0219, -0.0136, -0.0737,\n          0.1268,  0.0897, -0.1323,  0.0933, -0.0442,  0.0214,  0.0822,  0.1264,\n         -0.0975,  0.1420, -0.1363, -0.0658,  0.1223, -0.1284, -0.0184,  0.0267,\n         -0.1273,  0.0084,  0.0821,  0.0495,  0.1226,  0.1317, -0.1185, -0.0284,\n          0.0397, -0.0822, -0.0981, -0.0369, -0.0259,  0.1300,  0.0384,  0.0783,\n          0.0992]], device='cuda:0')), ('li.bias', tensor([-0.0468, -0.0961, -0.0226,  0.0455, -0.1347,  0.0440,  0.0832,  0.0816,\n        -0.0233, -0.1004], device='cuda:0'))])\n{'state': {}, 'param_groups': [{'lr': 0.05, 'momentum': 0, 'dampening': 0, 'weight_decay': 0, 'nesterov': False, 'initial_lr': 0.05, 'params': [2193304437528, 2193305078424, 2193311166536, 2193310019080, 2193310129912, 2193305190136]}]}\n"
    }
   ],
   "source": [
    "\n",
    "para_init = model.state_dict()\n",
    "optim_init = optimizer.state_dict()\n",
    "torch.save(model.state_dict(), 'model_init.pt')\n",
    "torch.save(optimizer.state_dict(), 'optim_init.pt')\n",
    "print(model.state_dict())\n",
    "print(optim_init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "iDz8qyb5SJjX",
    "outputId": "11a2f5d3-1943-4757-a4eb-b72d927dd40a",
    "tags": [
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend"
    ]
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "OrderedDict([('c1.weight', tensor([[[[ 0.0925, -0.0441,  0.0553],\n          [-0.1138, -0.2961,  0.0047],\n          [ 0.2043, -0.0859, -0.1000]]],\n\n\n        [[[ 0.2786, -0.2202, -0.1703],\n          [ 0.1581, -0.1463,  0.0819],\n          [-0.0489,  0.0269,  0.2253]]],\n\n\n        [[[ 0.0630, -0.2880, -0.3312],\n          [ 0.0864,  0.3215, -0.1820],\n          [-0.2329,  0.1246, -0.2132]]],\n\n\n        [[[-0.1638, -0.0027,  0.2143],\n          [ 0.1054,  0.2291, -0.0825],\n          [-0.0524, -0.0930, -0.1907]]],\n\n\n        [[[-0.0395, -0.1994,  0.0414],\n          [ 0.1200,  0.1663,  0.1609],\n          [-0.0673, -0.0838, -0.1943]]],\n\n\n        [[[ 0.2045,  0.0437, -0.2027],\n          [-0.3034, -0.2230, -0.2194],\n          [-0.1230,  0.1417,  0.2724]]]], device='cuda:0')), ('c1.bias', tensor([ 0.0712, -0.1130, -0.0558,  0.1180, -0.1398,  0.2255], device='cuda:0')), ('c2.weight', tensor([[[[-0.0188,  0.0184,  0.0924],\n          [ 0.0904, -0.0386, -0.0970],\n          [-0.1123, -0.0399, -0.1308]],\n\n         [[ 0.0316, -0.0420, -0.0796],\n          [-0.0241,  0.1242, -0.1160],\n          [ 0.1200, -0.0723, -0.0770]],\n\n         [[ 0.0819,  0.1143, -0.0121],\n          [ 0.0983, -0.0685, -0.1324],\n          [ 0.1236, -0.0944,  0.0295]],\n\n         [[ 0.0060,  0.0954, -0.0018],\n          [-0.1239,  0.1180, -0.0535],\n          [-0.0246, -0.0424, -0.0836]],\n\n         [[ 0.0952, -0.0344, -0.0372],\n          [ 0.0837, -0.0964, -0.0016],\n          [-0.0924, -0.0837,  0.0697]],\n\n         [[-0.0296, -0.0500,  0.0390],\n          [-0.0352, -0.1125,  0.0396],\n          [ 0.1076, -0.0536, -0.1119]]]], device='cuda:0')), ('c2.bias', tensor([-0.0129], device='cuda:0')), ('li.weight', tensor([[-0.0918, -0.0911,  0.0627, -0.1095, -0.0411,  0.0125, -0.0689,  0.0489,\n          0.0844,  0.1316, -0.1375, -0.0384, -0.1275,  0.0487, -0.1031, -0.1107,\n          0.1413,  0.0660, -0.0525, -0.0849, -0.0154,  0.0113, -0.1389, -0.0462,\n         -0.0896, -0.0157, -0.0989, -0.0819,  0.0680,  0.0535,  0.0251, -0.0148,\n          0.0872,  0.0552,  0.0337,  0.1364, -0.0334,  0.0922, -0.1169,  0.0665,\n          0.0224, -0.0623,  0.0546, -0.0758,  0.0277,  0.1098, -0.1174,  0.0328,\n         -0.0862],\n        [ 0.1056, -0.0422, -0.0716, -0.0061, -0.0806,  0.0621, -0.0376, -0.0857,\n          0.0902,  0.1257,  0.0466,  0.0312, -0.0220, -0.1240, -0.0361,  0.0486,\n         -0.0429,  0.0917,  0.0370, -0.0669, -0.0999, -0.0324,  0.1211,  0.0580,\n         -0.0079, -0.0259, -0.0652, -0.0719, -0.1317, -0.0434, -0.0519, -0.1402,\n         -0.0181,  0.0470,  0.0680, -0.0656, -0.0819, -0.0853, -0.1316,  0.1056,\n          0.0921, -0.1254,  0.1183,  0.1205,  0.0069,  0.0105, -0.1418, -0.0630,\n          0.1178],\n        [-0.0297, -0.0807,  0.0214, -0.1154, -0.0922, -0.1406,  0.0668,  0.0093,\n         -0.0487,  0.1307, -0.1133, -0.0962, -0.1415, -0.0291,  0.1244, -0.0158,\n         -0.1413,  0.1356, -0.0251,  0.0954,  0.0614,  0.1231, -0.0789,  0.0264,\n         -0.1039,  0.0286, -0.1143, -0.0907,  0.0016,  0.0406,  0.0052,  0.0870,\n          0.0958, -0.0596, -0.1050,  0.1328, -0.0062, -0.0857, -0.0343, -0.0729,\n          0.1339, -0.0710,  0.1018,  0.0026,  0.0579, -0.0894, -0.1315, -0.0686,\n          0.0887],\n        [ 0.0497,  0.1208,  0.1305, -0.1141,  0.1317, -0.1109, -0.1208,  0.1110,\n          0.0817, -0.0444, -0.0257,  0.0680, -0.0155,  0.0483,  0.0326,  0.0303,\n          0.0438,  0.0950, -0.1104, -0.0540, -0.0772, -0.0387,  0.0580, -0.1217,\n         -0.1185,  0.1358,  0.0870,  0.0099,  0.0388, -0.1085,  0.0493, -0.1011,\n         -0.0473,  0.0136, -0.1325,  0.0466,  0.0471, -0.0962, -0.0534,  0.0037,\n         -0.0802, -0.0656,  0.0817, -0.1179, -0.1341, -0.1183, -0.0977,  0.1011,\n          0.0430],\n        [-0.0243,  0.1397, -0.0901, -0.0640,  0.0024, -0.0176, -0.0539, -0.0644,\n          0.1269, -0.0066,  0.1158,  0.0521, -0.0554, -0.1268, -0.1125, -0.0598,\n         -0.1045,  0.0522, -0.0263, -0.1391, -0.1275,  0.1308, -0.1157, -0.0838,\n          0.0871, -0.1406, -0.1259,  0.0329, -0.0954, -0.0035,  0.1343, -0.1014,\n          0.0544, -0.0562,  0.0638, -0.0263, -0.0274,  0.1227, -0.0085, -0.0898,\n          0.0940,  0.0837, -0.0630, -0.1145, -0.1253,  0.1056,  0.1158, -0.0849,\n          0.0405],\n        [ 0.0366,  0.1240,  0.0844,  0.0808,  0.0178, -0.1103, -0.0482, -0.1045,\n         -0.1165, -0.0816,  0.0781,  0.0821, -0.0779,  0.0014,  0.0112,  0.0809,\n         -0.0176,  0.0382, -0.0111,  0.0589, -0.0825, -0.0175, -0.0004, -0.0024,\n          0.0436, -0.0174, -0.0201, -0.0032,  0.0534,  0.0923, -0.0357,  0.0731,\n          0.1065,  0.0219, -0.1300,  0.0959, -0.0436,  0.0521, -0.0586, -0.0101,\n         -0.0521,  0.0044, -0.0743,  0.0626, -0.0845, -0.1183, -0.0870, -0.0849,\n          0.1377],\n        [-0.0125, -0.0649, -0.0181,  0.0489, -0.0386, -0.0888, -0.0252, -0.0178,\n         -0.0183, -0.0915,  0.1263, -0.0701, -0.0959, -0.1202, -0.0185,  0.0549,\n          0.0119,  0.1057,  0.0744, -0.1341,  0.0408,  0.0816,  0.1331,  0.0009,\n         -0.1185,  0.1097, -0.0398,  0.0713,  0.1231, -0.0274,  0.1267, -0.0767,\n         -0.0304,  0.0934, -0.0603,  0.0609,  0.0317, -0.1182,  0.1070, -0.0883,\n         -0.0988,  0.0210,  0.0589, -0.1128,  0.1027, -0.0612,  0.0469,  0.0981,\n          0.1404],\n        [-0.0425,  0.0783, -0.1148, -0.0602, -0.0022, -0.0162,  0.0640, -0.0138,\n          0.0497,  0.0945, -0.0031,  0.1175,  0.0429, -0.0838, -0.1244,  0.1280,\n         -0.0002,  0.0941, -0.0543,  0.0855,  0.0261,  0.0084,  0.0065,  0.1019,\n         -0.0629, -0.1304,  0.0176, -0.0304,  0.0995,  0.0305, -0.0760,  0.0178,\n          0.0807, -0.1141,  0.1248,  0.1381, -0.0373, -0.0938,  0.1028,  0.0882,\n          0.1189, -0.1296, -0.0793, -0.0213,  0.0047, -0.0294, -0.0448, -0.1091,\n          0.0004],\n        [-0.0536, -0.0434, -0.0177, -0.0271, -0.1135, -0.0527, -0.0058, -0.0928,\n          0.1390,  0.0758, -0.0601,  0.1165, -0.1287,  0.0847, -0.0277,  0.1254,\n         -0.0569,  0.0889,  0.0913, -0.0753, -0.1329, -0.0126, -0.0957, -0.0768,\n          0.0190,  0.1283, -0.1204,  0.0294,  0.0788,  0.0769,  0.0933,  0.1232,\n         -0.0074, -0.0427, -0.0147, -0.0651,  0.0741,  0.1151, -0.0205,  0.0272,\n          0.0411,  0.0257,  0.1428, -0.1274,  0.0011, -0.0707, -0.0702, -0.0953,\n         -0.0754],\n        [ 0.0536, -0.0307,  0.0124, -0.1100, -0.0294,  0.0544, -0.1414,  0.0229,\n         -0.0982,  0.0929,  0.0510, -0.0535, -0.0489,  0.0219, -0.0136, -0.0737,\n          0.1268,  0.0897, -0.1323,  0.0933, -0.0442,  0.0214,  0.0822,  0.1264,\n         -0.0975,  0.1420, -0.1363, -0.0658,  0.1223, -0.1284, -0.0184,  0.0267,\n         -0.1273,  0.0084,  0.0821,  0.0495,  0.1226,  0.1317, -0.1185, -0.0284,\n          0.0397, -0.0822, -0.0981, -0.0369, -0.0259,  0.1300,  0.0384,  0.0783,\n          0.0992]], device='cuda:0')), ('li.bias', tensor([-0.0468, -0.0961, -0.0226,  0.0455, -0.1347,  0.0440,  0.0832,  0.0816,\n        -0.0233, -0.1004], device='cuda:0'))])\n==>>> epoch: 0, batch index: 1200, train loss: 2.301834\n==>>> epoch: 0, batch index: 200, test loss: 2.301179, acc: 0.114\n0.05\n50\n==>>> epoch: 1, batch index: 1200, train loss: 2.301434\n==>>> epoch: 1, batch index: 200, test loss: 2.301220, acc: 0.114\n0.05\n50\n==>>> epoch: 2, batch index: 1200, train loss: 2.301482\n==>>> epoch: 2, batch index: 200, test loss: 2.301261, acc: 0.114\n0.025\n50\n==>>> epoch: 3, batch index: 1200, train loss: 2.301322\n==>>> epoch: 3, batch index: 200, test loss: 2.301019, acc: 0.114\n0.025\n50\n==>>> epoch: 4, batch index: 1200, train loss: 2.301334\n==>>> epoch: 4, batch index: 200, test loss: 2.301039, acc: 0.114\n0.025\n50\n==>>> epoch: 5, batch index: 1200, train loss: 2.301350\n==>>> epoch: 5, batch index: 200, test loss: 2.300988, acc: 0.114\n0.0125\n50\n==>>> epoch: 6, batch index: 1200, train loss: 2.301243\n==>>> epoch: 6, batch index: 200, test loss: 2.300995, acc: 0.114\n0.0125\n50\n==>>> epoch: 7, batch index: 1200, train loss: 2.301260\n==>>> epoch: 7, batch index: 200, test loss: 2.301017, acc: 0.114\n0.0125\n50\n==>>> epoch: 8, batch index: 1200, train loss: 2.301262\n==>>> epoch: 8, batch index: 200, test loss: 2.300992, acc: 0.114\n0.00625\n50\n==>>> epoch: 9, batch index: 1200, train loss: 2.301213\n==>>> epoch: 9, batch index: 200, test loss: 2.300997, acc: 0.114\n0.00625\n50\n==>>> epoch: 10, batch index: 1200, train loss: 2.301212\n==>>> epoch: 10, batch index: 200, test loss: 2.301005, acc: 0.114\n0.00625\n50\n==>>> epoch: 11, batch index: 1200, train loss: 2.301214\n==>>> epoch: 11, batch index: 200, test loss: 2.301011, acc: 0.114\n0.003125\n50\n==>>> epoch: 12, batch index: 1200, train loss: 2.301187\n==>>> epoch: 12, batch index: 200, test loss: 2.301012, acc: 0.114\n0.003125\n50\n==>>> epoch: 13, batch index: 1200, train loss: 2.301186\n==>>> epoch: 13, batch index: 200, test loss: 2.301009, acc: 0.114\n0.003125\n50\n==>>> epoch: 14, batch index: 1200, train loss: 2.301186\n==>>> epoch: 14, batch index: 200, test loss: 2.301013, acc: 0.114\n0.0015625\n50\n==>>> epoch: 15, batch index: 1200, train loss: 2.301173\n==>>> epoch: 15, batch index: 200, test loss: 2.301015, acc: 0.114\n0.0015625\n50\n==>>> epoch: 16, batch index: 1200, train loss: 2.301173\n==>>> epoch: 16, batch index: 200, test loss: 2.301016, acc: 0.114\n0.0015625\n50\n==>>> epoch: 17, batch index: 1200, train loss: 2.301173\n==>>> epoch: 17, batch index: 200, test loss: 2.301016, acc: 0.114\n0.00078125\n50\n==>>> epoch: 18, batch index: 1200, train loss: 2.301166\n==>>> epoch: 18, batch index: 200, test loss: 2.301017, acc: 0.114\n0.00078125\n50\n==>>> epoch: 19, batch index: 1200, train loss: 2.301166\n==>>> epoch: 19, batch index: 200, test loss: 2.301017, acc: 0.114\n0.00078125\n50\n==>>> epoch: 20, batch index: 1200, train loss: 2.301166\n==>>> epoch: 20, batch index: 200, test loss: 2.301017, acc: 0.114\n0.000390625\n50\n==>>> epoch: 21, batch index: 1200, train loss: 2.301163\n==>>> epoch: 21, batch index: 200, test loss: 2.301017, acc: 0.114\n0.000390625\n50\n==>>> epoch: 22, batch index: 1200, train loss: 2.301163\n==>>> epoch: 22, batch index: 200, test loss: 2.301018, acc: 0.114\n0.000390625\n50\n==>>> epoch: 23, batch index: 1200, train loss: 2.301163\n==>>> epoch: 23, batch index: 200, test loss: 2.301017, acc: 0.114\n0.0001953125\n50\n==>>> epoch: 24, batch index: 1200, train loss: 2.301161\n==>>> epoch: 24, batch index: 200, test loss: 2.301017, acc: 0.114\n0.0001953125\n50\n==>>> epoch: 25, batch index: 1200, train loss: 2.301161\n==>>> epoch: 25, batch index: 200, test loss: 2.301017, acc: 0.114\n0.0001953125\n50\n==>>> epoch: 26, batch index: 1200, train loss: 2.301161\n==>>> epoch: 26, batch index: 200, test loss: 2.301017, acc: 0.114\n9.765625e-05\n50\n==>>> epoch: 27, batch index: 1200, train loss: 2.301160\n==>>> epoch: 27, batch index: 200, test loss: 2.301018, acc: 0.114\n9.765625e-05\n50\n==>>> epoch: 28, batch index: 1200, train loss: 2.301160\n==>>> epoch: 28, batch index: 200, test loss: 2.301018, acc: 0.114\n9.765625e-05\n50\n==>>> epoch: 29, batch index: 1200, train loss: 2.301160\n==>>> epoch: 29, batch index: 200, test loss: 2.301017, acc: 0.114\n4.8828125e-05\n50\n"
    }
   ],
   "source": [
    "# model.load_state_dict(torch.load('model_init.pt'))\n",
    "# optimizer.load_state_dict(torch.load('optim_init.pt'))\n",
    "\n",
    "# model.load_state_dict(para_init)\n",
    "print(model.state_dict())\n",
    "lrd_train_loss = []\n",
    "lrd_test_acc = []\n",
    "for epoch in range(total_epoch):\n",
    "    # trainning\n",
    "    total_loss = 0\n",
    "    total_batch = 0\n",
    "    for batch_idx, (x, target) in enumerate(train_loader):\n",
    "        model.train()\n",
    "        optimizer.zero_grad() # [Question 7] What is 'optimizer.zero_grad()' used for?\n",
    "        if use_cuda:\n",
    "            x, target = x.cuda(), target.cuda()\n",
    "            \n",
    "        out = model(x)\n",
    "        loss = criterion(out, target) # [Question 8] What are 'out' and 'target' and what is output of criterion(out, target)?\n",
    "\n",
    "        total_loss += loss.item()\n",
    "        total_batch += 1\n",
    "        loss.backward() # [Question 9] What is 'loss.backward()' used for?\n",
    "        optimizer.step() # [Question 10] What is 'optimizer.step()' used for?\n",
    "    print ('==>>> epoch: {}, batch index: {}, train loss: {:.6f}'\n",
    "            .format(epoch, batch_idx+1, total_loss / total_batch))\n",
    "    lrd_train_loss.append(total_loss / total_batch)\n",
    "    \n",
    "    # testing\n",
    "    total_loss = 0\n",
    "    total_batch = 0\n",
    "    correct_cnt = 0\n",
    "    total_cnt = 0\n",
    "    \n",
    "    for batch_idx, (x, target) in enumerate(test_loader):\n",
    "        model.eval()\n",
    "        if use_cuda:\n",
    "            x, target = x.cuda(), target.cuda()\n",
    "            \n",
    "        out = model(x)\n",
    "        loss = criterion(out, target)\n",
    "        _, pred_label = torch.max(out.data, 1)\n",
    "        total_cnt += x.data.size()[0]\n",
    "        correct_cnt += (pred_label == target.data).sum().item()\n",
    "        \n",
    "        total_loss += loss.item()\n",
    "        total_batch += 1\n",
    "        \n",
    "    print ('==>>> epoch: {}, batch index: {}, test loss: {:.6f}, acc: {:.3f}'\n",
    "            .format(epoch, batch_idx+1, total_loss / total_batch, correct_cnt * 1.0 / total_cnt))\n",
    "    lrd_test_acc.append(correct_cnt * 1.0 / total_cnt)\n",
    "    lr_scheduler.step()\n",
    "    for param_group in optimizer.param_groups:\n",
    "        print(param_group['lr'])\n",
    "\n",
    "    print(batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "tags": [
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend",
     "outputPrepend"
    ]
   },
   "outputs": [
    {
     "output_type": "stream",
     "name": "stdout",
     "text": "-02,  1.1315e-01, -1.4223e-01, -4.4571e-02],\n        [ 9.7951e-02, -1.3406e-01, -3.3114e-02,  5.9820e-02,  2.7808e-02,\n         -6.8903e-02, -6.0153e-02, -8.8767e-02, -1.0317e-01,  8.6698e-02,\n          5.5030e-02, -1.3284e-01,  6.9594e-02,  3.4479e-02,  5.9333e-02,\n         -9.6490e-02,  2.7401e-02,  9.5722e-02, -2.3817e-03,  8.7552e-02,\n          1.2327e-01,  1.1316e-01,  4.7774e-02,  2.4526e-03, -6.2129e-02,\n          5.3357e-02, -8.7845e-02,  7.9004e-02,  2.8070e-02,  8.5615e-02,\n          6.6302e-02, -1.0592e-01, -1.2879e-01,  3.4069e-03, -5.5848e-02,\n         -1.2851e-01, -1.2990e-01, -3.4027e-02, -5.3044e-02,  8.2646e-02,\n          9.6923e-02,  4.2569e-03, -1.2952e-01, -8.5417e-02, -5.0403e-02,\n         -6.5623e-02,  1.3446e-01,  9.2206e-02,  9.8122e-02],\n        [-1.0480e-01, -4.2144e-02, -1.2213e-01,  1.1575e-01,  6.7105e-02,\n          9.2445e-02, -5.4336e-02, -1.2235e-01,  1.3456e-01, -1.1189e-01,\n          7.1115e-02, -8.4091e-03,  7.7561e-02,  4.6410e-02,  4.3646e-03,\n         -1.3474e-01,  3.6932e-02, -9.5503e-02,  1.3494e-01,  7.7154e-02,\n          4.8406e-02, -4.5881e-02,  9.2358e-02,  7.2855e-02,  5.5936e-02,\n          1.1462e-01,  6.1378e-02, -3.7936e-02,  1.4207e-01,  1.1544e-01,\n          6.6540e-02, -1.2576e-01,  4.3452e-02,  7.5930e-02, -7.6085e-03,\n         -5.2219e-03, -6.8618e-02,  1.2918e-01,  1.0641e-01,  1.2551e-01,\n          1.0523e-02,  1.3888e-01, -6.0823e-02,  1.6717e-02,  5.9744e-02,\n         -6.0661e-02,  6.4456e-02, -1.2847e-01,  6.3170e-03],\n        [-9.7726e-02,  1.4026e-01,  6.9751e-02, -2.4547e-02,  5.1203e-02,\n          3.1536e-02,  3.6413e-03,  1.3231e-01, -6.8723e-03,  1.0201e-01,\n          1.3344e-01, -1.3766e-02,  3.1414e-02,  1.2766e-01,  4.6003e-02,\n         -2.9802e-02, -6.9304e-02, -7.0351e-02, -1.0250e-01,  6.8808e-02,\n          5.2015e-02, -1.1182e-02, -1.3446e-01, -7.1316e-02, -6.8842e-02,\n          1.8359e-02,  3.2452e-02,  2.3678e-02, -1.0712e-01,  1.3010e-01,\n         -3.8907e-03, -1.1799e-01,  1.3304e-01,  7.6018e-02, -1.4059e-01,\n          1.4129e-01, -1.1486e-02,  5.6021e-02, -1.1347e-01,  1.6226e-02,\n          5.2856e-02, -2.0396e-02,  1.3419e-01, -1.0499e-01, -3.6625e-03,\n          7.5608e-02,  6.6312e-02, -3.1301e-02, -1.3330e-01],\n        [ 1.2268e-01, -1.0574e-01, -1.1657e-01, -4.7841e-02, -8.6484e-02,\n         -7.2581e-02, -9.1194e-02,  4.2765e-02,  4.5154e-02,  4.5020e-02,\n         -1.3962e-01,  1.0265e-01,  1.1874e-01,  4.3440e-02,  1.0557e-01,\n         -4.4122e-02,  5.8834e-02,  8.8848e-02,  5.8197e-02, -1.7326e-02,\n         -2.4043e-02, -2.2696e-02,  6.0809e-02,  3.9355e-04, -2.2587e-02,\n         -1.3874e-01, -3.8119e-03, -5.7031e-02,  1.2314e-01,  4.3343e-02,\n          6.6679e-02,  1.2479e-01, -4.1594e-02, -1.3777e-01, -3.0715e-02,\n         -8.2316e-02, -5.6043e-03,  1.3892e-01, -9.6528e-03, -1.1608e-01,\n         -8.3983e-03, -9.1852e-02,  8.8207e-02,  1.0618e-02,  7.0809e-02,\n          4.4727e-02,  1.0174e-01,  1.2538e-01, -9.0049e-02],\n        [ 8.4397e-02,  4.2218e-02,  1.0083e-01, -1.2096e-01,  8.7890e-02,\n         -2.4771e-02,  8.0861e-02, -1.2567e-01,  9.8439e-03,  2.7014e-02,\n         -1.4023e-01,  1.2037e-01, -8.0973e-03,  7.7879e-02,  5.4811e-02,\n          1.1461e-01,  1.0549e-01,  1.3295e-01, -8.7652e-04,  6.4751e-02,\n         -2.1576e-02,  1.8893e-03,  5.9379e-02, -1.1239e-01, -5.3765e-03,\n          4.2981e-02, -2.2137e-02, -6.8451e-02, -4.5930e-02, -3.4137e-03,\n          1.0167e-01,  1.3840e-01, -6.1369e-03, -1.2009e-01,  1.4983e-02,\n          1.3656e-01, -1.0481e-01,  1.3468e-01,  1.1271e-01,  1.3015e-01,\n         -8.0021e-02, -1.3865e-01,  3.7601e-03,  1.0340e-01, -1.2776e-02,\n          1.2193e-02,  6.4944e-02,  1.1776e-01,  1.0844e-01],\n        [ 9.1504e-02, -1.5577e-02, -8.3069e-02,  2.2190e-02, -1.1796e-02,\n          1.0397e-01,  1.2085e-05,  2.5170e-02,  2.0811e-03,  6.6192e-03,\n         -8.8484e-02, -3.7947e-02,  3.1557e-02,  1.2570e-01,  8.8107e-02,\n          2.3997e-02, -6.0844e-02, -5.7214e-02,  9.7576e-02, -4.4774e-02,\n          1.0496e-01, -5.5658e-02,  8.9052e-03,  7.1512e-02,  1.1336e-01,\n         -2.4567e-02, -8.5833e-02,  5.5413e-02,  2.7927e-02, -8.9631e-02,\n         -1.3550e-01, -8.5353e-02, -5.1845e-02, -5.6238e-02,  3.6043e-02,\n         -5.3467e-02, -8.2081e-02,  8.9116e-02, -1.2335e-01,  3.0059e-02,\n         -3.7826e-02, -4.8167e-02,  1.0940e-01,  6.4133e-02,  1.4177e-01,\n          1.2855e-01, -1.0196e-01,  7.2034e-02, -1.2739e-01],\n        [-7.8865e-02, -3.2270e-02,  1.1800e-01,  6.0124e-02, -4.1585e-02,\n          8.2370e-02, -6.7509e-02, -1.7950e-02,  1.9827e-02,  9.1037e-02,\n         -1.1847e-01,  9.8481e-02,  1.0063e-01, -1.2975e-02, -3.2811e-02,\n          7.4822e-02,  1.3118e-01, -8.1797e-02, -6.4915e-02,  7.6015e-02,\n         -9.4298e-02, -1.2100e-01,  1.0781e-01, -7.3356e-02, -8.8875e-02,\n          6.6650e-02,  5.9961e-02, -5.0935e-03,  5.1092e-02,  1.0857e-01,\n          1.2791e-01, -5.2343e-02, -1.2174e-01,  5.1809e-02,  3.4074e-02,\n          7.5739e-02,  2.3746e-02,  1.2743e-01,  1.8314e-02, -1.0705e-01,\n          5.9538e-02,  1.2583e-01, -1.6069e-02, -1.1171e-01,  1.2812e-01,\n         -1.3547e-01, -8.5901e-02, -1.0318e-01,  3.2890e-03],\n        [-8.9207e-02, -5.9766e-02, -1.0670e-01, -1.3143e-02, -7.1515e-02,\n          1.3185e-01,  7.1570e-02,  1.0771e-01, -9.1727e-02,  1.1957e-01,\n          1.0335e-01,  6.6116e-02, -5.5243e-02,  1.2454e-01,  3.1224e-02,\n         -7.6854e-02,  9.5382e-02,  4.2971e-02,  4.4566e-03, -7.9373e-02,\n          1.2588e-01,  4.5748e-02, -1.2176e-01, -4.9090e-02, -1.3059e-01,\n         -1.1880e-01, -1.0312e-01,  1.1719e-01,  1.3509e-01, -1.3710e-01,\n          1.2149e-01, -2.3159e-02,  2.5702e-02,  6.0582e-02, -2.8036e-02,\n          1.8984e-02, -1.3493e-01,  5.3906e-02, -2.6498e-02,  5.1713e-02,\n         -1.1673e-01,  1.4140e-01, -6.7010e-02, -5.8798e-02, -3.8865e-02,\n          2.3193e-02,  1.0336e-01, -8.9838e-02,  3.3169e-02],\n        [-1.3073e-01, -6.7250e-02,  2.7201e-02, -9.3367e-02, -5.4071e-03,\n         -3.9891e-03, -1.1755e-01, -5.1902e-02, -1.9649e-03, -1.8439e-02,\n         -1.1494e-01,  2.5780e-02, -1.1704e-01,  7.6939e-02,  9.0031e-03,\n         -1.7519e-02,  1.2110e-01, -4.0490e-02,  6.6944e-02,  6.0878e-02,\n          3.8886e-02,  7.5228e-02, -2.4205e-03, -4.7660e-02,  1.2559e-01,\n          1.1852e-01,  1.1951e-01,  3.2366e-02, -3.0646e-02, -6.0633e-02,\n          1.5560e-02,  8.0222e-03, -5.7153e-02, -1.3778e-01,  3.6487e-02,\n          5.7323e-02,  1.4089e-01, -3.5250e-02,  1.0150e-01, -3.2563e-02,\n          7.1999e-02, -7.5014e-02,  1.2923e-01,  1.2908e-01, -8.8809e-02,\n         -7.5870e-02, -1.3300e-01,  5.2282e-02, -1.3208e-01]], device='cuda:0')), ('li.bias', tensor([-0.0167, -0.0198, -0.0025, -0.0009, -0.0439, -0.0223,  0.0315, -0.0738,\n         0.0195, -0.1301], device='cuda:0'))])\n{'state': {}, 'param_groups': [{'lr': 0.05, 'momentum': 0, 'dampening': 0, 'weight_decay': 0, 'nesterov': False, 'initial_lr': 0.05, 'params': [2193310109192, 2193310105672, 2193310107592, 2193310105912, 2193310109432, 2193310107752]}]}\nOrderedDict([('c1.weight', tensor([[[[ 0.0386, -0.2937, -0.1324],\n          [ 0.1337,  0.2549,  0.1895],\n          [-0.2249, -0.2648,  0.0792]]],\n\n\n        [[[ 0.3202, -0.2991,  0.0547],\n          [-0.1470, -0.0908,  0.1983],\n          [-0.1094,  0.1131, -0.0261]]],\n\n\n        [[[ 0.2274, -0.0429,  0.2633],\n          [-0.2409,  0.1088,  0.2781],\n          [ 0.1509, -0.2899,  0.1135]]],\n\n\n        [[[-0.2991,  0.1473,  0.0778],\n          [-0.0964,  0.0865, -0.1631],\n          [ 0.2649,  0.3078, -0.2520]]],\n\n\n        [[[ 0.1747, -0.1248, -0.2931],\n          [-0.0118,  0.3046, -0.0090],\n          [ 0.0656, -0.2828,  0.3049]]],\n\n\n        [[[ 0.1090, -0.0223,  0.2954],\n          [ 0.1624, -0.1878, -0.2895],\n          [ 0.0503, -0.0120, -0.0424]]]], device='cuda:0')), ('c1.bias', tensor([ 0.1537, -0.0937, -0.2974, -0.1234,  0.0961, -0.1176], device='cuda:0')), ('c2.weight', tensor([[[[ 0.0843, -0.0922,  0.1013],\n          [-0.0059,  0.0106,  0.1344],\n          [ 0.0827,  0.0592,  0.0747]],\n\n         [[ 0.0186, -0.0914,  0.0061],\n          [-0.1313,  0.0024, -0.0964],\n          [-0.0310,  0.0553, -0.1339]],\n\n         [[-0.1255, -0.0944, -0.0660],\n          [-0.0986,  0.0437, -0.0596],\n          [ 0.1188, -0.0958, -0.1121]],\n\n         [[-0.0921, -0.0351,  0.0061],\n          [ 0.1201, -0.0104,  0.0830],\n          [ 0.0718, -0.0226,  0.0843]],\n\n         [[-0.0007, -0.0485, -0.1245],\n          [ 0.1044,  0.0585, -0.0911],\n          [-0.0998, -0.1073,  0.0203]],\n\n         [[ 0.0361, -0.1279,  0.0593],\n          [-0.1221, -0.1036,  0.0731],\n          [-0.1171, -0.0659,  0.0547]]]], device='cuda:0')), ('c2.bias', tensor([-0.1335], device='cuda:0')), ('li.weight', tensor([[-1.0035e-03, -1.3033e-03,  8.1144e-02, -1.1045e-01, -5.2916e-02,\n          7.5819e-02, -1.1034e-01, -7.7370e-02,  8.8023e-02, -9.7189e-02,\n         -8.2644e-02, -1.0410e-01,  1.0838e-01, -2.1718e-02,  1.2143e-01,\n         -5.0624e-02,  2.1395e-02,  2.9209e-02,  1.0481e-01, -1.1431e-01,\n         -6.8455e-02, -9.3458e-02, -2.8240e-02, -5.1067e-02, -1.2521e-01,\n          1.2064e-01, -1.2278e-01,  1.2592e-01,  1.4001e-01, -6.4706e-02,\n          8.5772e-03,  7.9325e-02,  6.0783e-02,  3.0079e-02,  8.6194e-02,\n         -5.4113e-02,  1.2538e-01, -8.1786e-02,  4.3361e-02,  6.9542e-02,\n          9.8653e-02, -7.1752e-02,  9.4167e-02, -3.9689e-02,  9.8319e-02,\n         -7.8381e-02,  1.1315e-01, -1.4223e-01, -4.4571e-02],\n        [ 9.7951e-02, -1.3406e-01, -3.3114e-02,  5.9820e-02,  2.7808e-02,\n         -6.8903e-02, -6.0153e-02, -8.8767e-02, -1.0317e-01,  8.6698e-02,\n          5.5030e-02, -1.3284e-01,  6.9594e-02,  3.4479e-02,  5.9333e-02,\n         -9.6490e-02,  2.7401e-02,  9.5722e-02, -2.3817e-03,  8.7552e-02,\n          1.2327e-01,  1.1316e-01,  4.7774e-02,  2.4526e-03, -6.2129e-02,\n          5.3357e-02, -8.7845e-02,  7.9004e-02,  2.8070e-02,  8.5615e-02,\n          6.6302e-02, -1.0592e-01, -1.2879e-01,  3.4069e-03, -5.5848e-02,\n         -1.2851e-01, -1.2990e-01, -3.4027e-02, -5.3044e-02,  8.2646e-02,\n          9.6923e-02,  4.2569e-03, -1.2952e-01, -8.5417e-02, -5.0403e-02,\n         -6.5623e-02,  1.3446e-01,  9.2206e-02,  9.8122e-02],\n        [-1.0480e-01, -4.2144e-02, -1.2213e-01,  1.1575e-01,  6.7105e-02,\n          9.2445e-02, -5.4336e-02, -1.2235e-01,  1.3456e-01, -1.1189e-01,\n          7.1115e-02, -8.4091e-03,  7.7561e-02,  4.6410e-02,  4.3646e-03,\n         -1.3474e-01,  3.6932e-02, -9.5503e-02,  1.3494e-01,  7.7154e-02,\n          4.8406e-02, -4.5881e-02,  9.2358e-02,  7.2855e-02,  5.5936e-02,\n          1.1462e-01,  6.1378e-02, -3.7936e-02,  1.4207e-01,  1.1544e-01,\n          6.6540e-02, -1.2576e-01,  4.3452e-02,  7.5930e-02, -7.6085e-03,\n         -5.2219e-03, -6.8618e-02,  1.2918e-01,  1.0641e-01,  1.2551e-01,\n          1.0523e-02,  1.3888e-01, -6.0823e-02,  1.6717e-02,  5.9744e-02,\n         -6.0661e-02,  6.4456e-02, -1.2847e-01,  6.3170e-03],\n        [-9.7726e-02,  1.4026e-01,  6.9751e-02, -2.4547e-02,  5.1203e-02,\n          3.1536e-02,  3.6413e-03,  1.3231e-01, -6.8723e-03,  1.0201e-01,\n          1.3344e-01, -1.3766e-02,  3.1414e-02,  1.2766e-01,  4.6003e-02,\n         -2.9802e-02, -6.9304e-02, -7.0351e-02, -1.0250e-01,  6.8808e-02,\n          5.2015e-02, -1.1182e-02, -1.3446e-01, -7.1316e-02, -6.8842e-02,\n          1.8359e-02,  3.2452e-02,  2.3678e-02, -1.0712e-01,  1.3010e-01,\n         -3.8907e-03, -1.1799e-01,  1.3304e-01,  7.6018e-02, -1.4059e-01,\n          1.4129e-01, -1.1486e-02,  5.6021e-02, -1.1347e-01,  1.6226e-02,\n          5.2856e-02, -2.0396e-02,  1.3419e-01, -1.0499e-01, -3.6625e-03,\n          7.5608e-02,  6.6312e-02, -3.1301e-02, -1.3330e-01],\n        [ 1.2268e-01, -1.0574e-01, -1.1657e-01, -4.7841e-02, -8.6484e-02,\n         -7.2581e-02, -9.1194e-02,  4.2765e-02,  4.5154e-02,  4.5020e-02,\n         -1.3962e-01,  1.0265e-01,  1.1874e-01,  4.3440e-02,  1.0557e-01,\n         -4.4122e-02,  5.8834e-02,  8.8848e-02,  5.8197e-02, -1.7326e-02,\n         -2.4043e-02, -2.2696e-02,  6.0809e-02,  3.9355e-04, -2.2587e-02,\n         -1.3874e-01, -3.8119e-03, -5.7031e-02,  1.2314e-01,  4.3343e-02,\n          6.6679e-02,  1.2479e-01, -4.1594e-02, -1.3777e-01, -3.0715e-02,\n         -8.2316e-02, -5.6043e-03,  1.3892e-01, -9.6528e-03, -1.1608e-01,\n         -8.3983e-03, -9.1852e-02,  8.8207e-02,  1.0618e-02,  7.0809e-02,\n          4.4727e-02,  1.0174e-01,  1.2538e-01, -9.0049e-02],\n        [ 8.4397e-02,  4.2218e-02,  1.0083e-01, -1.2096e-01,  8.7890e-02,\n         -2.4771e-02,  8.0861e-02, -1.2567e-01,  9.8439e-03,  2.7014e-02,\n         -1.4023e-01,  1.2037e-01, -8.0973e-03,  7.7879e-02,  5.4811e-02,\n          1.1461e-01,  1.0549e-01,  1.3295e-01, -8.7652e-04,  6.4751e-02,\n         -2.1576e-02,  1.8893e-03,  5.9379e-02, -1.1239e-01, -5.3765e-03,\n          4.2981e-02, -2.2137e-02, -6.8451e-02, -4.5930e-02, -3.4137e-03,\n          1.0167e-01,  1.3840e-01, -6.1369e-03, -1.2009e-01,  1.4983e-02,\n          1.3656e-01, -1.0481e-01,  1.3468e-01,  1.1271e-01,  1.3015e-01,\n         -8.0021e-02, -1.3865e-01,  3.7601e-03,  1.0340e-01, -1.2776e-02,\n          1.2193e-02,  6.4944e-02,  1.1776e-01,  1.0844e-01],\n        [ 9.1504e-02, -1.5577e-02, -8.3069e-02,  2.2190e-02, -1.1796e-02,\n          1.0397e-01,  1.2085e-05,  2.5170e-02,  2.0811e-03,  6.6192e-03,\n         -8.8484e-02, -3.7947e-02,  3.1557e-02,  1.2570e-01,  8.8107e-02,\n          2.3997e-02, -6.0844e-02, -5.7214e-02,  9.7576e-02, -4.4774e-02,\n          1.0496e-01, -5.5658e-02,  8.9052e-03,  7.1512e-02,  1.1336e-01,\n         -2.4567e-02, -8.5833e-02,  5.5413e-02,  2.7927e-02, -8.9631e-02,\n         -1.3550e-01, -8.5353e-02, -5.1845e-02, -5.6238e-02,  3.6043e-02,\n         -5.3467e-02, -8.2081e-02,  8.9116e-02, -1.2335e-01,  3.0059e-02,\n         -3.7826e-02, -4.8167e-02,  1.0940e-01,  6.4133e-02,  1.4177e-01,\n          1.2855e-01, -1.0196e-01,  7.2034e-02, -1.2739e-01],\n        [-7.8865e-02, -3.2270e-02,  1.1800e-01,  6.0124e-02, -4.1585e-02,\n          8.2370e-02, -6.7509e-02, -1.7950e-02,  1.9827e-02,  9.1037e-02,\n         -1.1847e-01,  9.8481e-02,  1.0063e-01, -1.2975e-02, -3.2811e-02,\n          7.4822e-02,  1.3118e-01, -8.1797e-02, -6.4915e-02,  7.6015e-02,\n         -9.4298e-02, -1.2100e-01,  1.0781e-01, -7.3356e-02, -8.8875e-02,\n          6.6650e-02,  5.9961e-02, -5.0935e-03,  5.1092e-02,  1.0857e-01,\n          1.2791e-01, -5.2343e-02, -1.2174e-01,  5.1809e-02,  3.4074e-02,\n          7.5739e-02,  2.3746e-02,  1.2743e-01,  1.8314e-02, -1.0705e-01,\n          5.9538e-02,  1.2583e-01, -1.6069e-02, -1.1171e-01,  1.2812e-01,\n         -1.3547e-01, -8.5901e-02, -1.0318e-01,  3.2890e-03],\n        [-8.9207e-02, -5.9766e-02, -1.0670e-01, -1.3143e-02, -7.1515e-02,\n          1.3185e-01,  7.1570e-02,  1.0771e-01, -9.1727e-02,  1.1957e-01,\n          1.0335e-01,  6.6116e-02, -5.5243e-02,  1.2454e-01,  3.1224e-02,\n         -7.6854e-02,  9.5382e-02,  4.2971e-02,  4.4566e-03, -7.9373e-02,\n          1.2588e-01,  4.5748e-02, -1.2176e-01, -4.9090e-02, -1.3059e-01,\n         -1.1880e-01, -1.0312e-01,  1.1719e-01,  1.3509e-01, -1.3710e-01,\n          1.2149e-01, -2.3159e-02,  2.5702e-02,  6.0582e-02, -2.8036e-02,\n          1.8984e-02, -1.3493e-01,  5.3906e-02, -2.6498e-02,  5.1713e-02,\n         -1.1673e-01,  1.4140e-01, -6.7010e-02, -5.8798e-02, -3.8865e-02,\n          2.3193e-02,  1.0336e-01, -8.9838e-02,  3.3169e-02],\n        [-1.3073e-01, -6.7250e-02,  2.7201e-02, -9.3367e-02, -5.4071e-03,\n         -3.9891e-03, -1.1755e-01, -5.1902e-02, -1.9649e-03, -1.8439e-02,\n         -1.1494e-01,  2.5780e-02, -1.1704e-01,  7.6939e-02,  9.0031e-03,\n         -1.7519e-02,  1.2110e-01, -4.0490e-02,  6.6944e-02,  6.0878e-02,\n          3.8886e-02,  7.5228e-02, -2.4205e-03, -4.7660e-02,  1.2559e-01,\n          1.1852e-01,  1.1951e-01,  3.2366e-02, -3.0646e-02, -6.0633e-02,\n          1.5560e-02,  8.0222e-03, -5.7153e-02, -1.3778e-01,  3.6487e-02,\n          5.7323e-02,  1.4089e-01, -3.5250e-02,  1.0150e-01, -3.2563e-02,\n          7.1999e-02, -7.5014e-02,  1.2923e-01,  1.2908e-01, -8.8809e-02,\n         -7.5870e-02, -1.3300e-01,  5.2282e-02, -1.3208e-01]], device='cuda:0')), ('li.bias', tensor([-0.0167, -0.0198, -0.0025, -0.0009, -0.0439, -0.0223,  0.0315, -0.0738,\n         0.0195, -0.1301], device='cuda:0'))])\n==>>> epoch: 0, batch index: 600, train loss: 1.365982\n==>>> epoch: 0, batch index: 100, test loss: 0.462169, acc: 0.858\n0.05\n100\n==>>> epoch: 1, batch index: 600, train loss: 0.474524\n==>>> epoch: 1, batch index: 100, test loss: 0.411501, acc: 0.876\n0.05\n100\n==>>> epoch: 2, batch index: 600, train loss: 0.430759\n==>>> epoch: 2, batch index: 100, test loss: 0.405057, acc: 0.875\n0.05\n100\n==>>> epoch: 3, batch index: 300, train loss: 0.396319\n==>>> epoch: 3, batch index: 50, test loss: 0.364307, acc: 0.891\n0.05\n200\n==>>> epoch: 4, batch index: 300, train loss: 0.387629\n==>>> epoch: 4, batch index: 50, test loss: 0.363818, acc: 0.890\n0.05\n200\n==>>> epoch: 5, batch index: 300, train loss: 0.380852\n==>>> epoch: 5, batch index: 50, test loss: 0.350144, acc: 0.896\n0.05\n200\n==>>> epoch: 6, batch index: 150, train loss: 0.364250\n==>>> epoch: 6, batch index: 25, test loss: 0.346584, acc: 0.895\n0.05\n400\n==>>> epoch: 7, batch index: 150, train loss: 0.362314\n==>>> epoch: 7, batch index: 25, test loss: 0.340020, acc: 0.898\n0.05\n400\n==>>> epoch: 8, batch index: 150, train loss: 0.359701\n==>>> epoch: 8, batch index: 25, test loss: 0.343866, acc: 0.896\n0.05\n400\n==>>> epoch: 9, batch index: 75, train loss: 0.351626\n==>>> epoch: 9, batch index: 13, test loss: 0.340992, acc: 0.900\n0.05\n800\n==>>> epoch: 10, batch index: 75, train loss: 0.349533\n==>>> epoch: 10, batch index: 13, test loss: 0.347219, acc: 0.896\n0.05\n800\n==>>> epoch: 11, batch index: 75, train loss: 0.347884\n==>>> epoch: 11, batch index: 13, test loss: 0.345335, acc: 0.900\n0.05\n800\n==>>> epoch: 12, batch index: 75, train loss: 0.346234\n==>>> epoch: 12, batch index: 13, test loss: 0.337763, acc: 0.900\n0.05\n800\n==>>> epoch: 13, batch index: 75, train loss: 0.345368\n==>>> epoch: 13, batch index: 13, test loss: 0.351389, acc: 0.895\n0.05\n800\n==>>> epoch: 14, batch index: 75, train loss: 0.343903\n==>>> epoch: 14, batch index: 13, test loss: 0.330420, acc: 0.903\n0.05\n800\n==>>> epoch: 15, batch index: 75, train loss: 0.343601\n==>>> epoch: 15, batch index: 13, test loss: 0.330460, acc: 0.903\n0.05\n800\n==>>> epoch: 16, batch index: 75, train loss: 0.340176\n==>>> epoch: 16, batch index: 13, test loss: 0.330601, acc: 0.901\n0.05\n800\n==>>> epoch: 17, batch index: 75, train loss: 0.338706\n==>>> epoch: 17, batch index: 13, test loss: 0.345432, acc: 0.892\n0.05\n800\n==>>> epoch: 18, batch index: 75, train loss: 0.336791\n==>>> epoch: 18, batch index: 13, test loss: 0.331488, acc: 0.899\n0.05\n800\n==>>> epoch: 19, batch index: 75, train loss: 0.337392\n==>>> epoch: 19, batch index: 13, test loss: 0.323796, acc: 0.905\n0.05\n800\n==>>> epoch: 20, batch index: 75, train loss: 0.336806\n==>>> epoch: 20, batch index: 13, test loss: 0.324178, acc: 0.904\n0.05\n800\n==>>> epoch: 21, batch index: 75, train loss: 0.332266\n==>>> epoch: 21, batch index: 13, test loss: 0.320179, acc: 0.906\n0.05\n800\n==>>> epoch: 22, batch index: 75, train loss: 0.330891\n==>>> epoch: 22, batch index: 13, test loss: 0.316648, acc: 0.906\n0.05\n800\n==>>> epoch: 23, batch index: 75, train loss: 0.329544\n==>>> epoch: 23, batch index: 13, test loss: 0.317377, acc: 0.906\n0.05\n800\n==>>> epoch: 24, batch index: 75, train loss: 0.327940\n==>>> epoch: 24, batch index: 13, test loss: 0.323901, acc: 0.902\n0.05\n800\n==>>> epoch: 25, batch index: 75, train loss: 0.325267\n==>>> epoch: 25, batch index: 13, test loss: 0.325898, acc: 0.900\n0.05\n800\n==>>> epoch: 26, batch index: 75, train loss: 0.325127\n==>>> epoch: 26, batch index: 13, test loss: 0.310766, acc: 0.907\n0.05\n800\n==>>> epoch: 27, batch index: 75, train loss: 0.320074\n==>>> epoch: 27, batch index: 13, test loss: 0.311820, acc: 0.908\n0.05\n800\n==>>> epoch: 28, batch index: 75, train loss: 0.324366\n==>>> epoch: 28, batch index: 13, test loss: 0.322238, acc: 0.902\n0.05\n800\n==>>> epoch: 29, batch index: 75, train loss: 0.318476\n==>>> epoch: 29, batch index: 13, test loss: 0.302825, acc: 0.910\n0.05\n800\n"
    }
   ],
   "source": [
    "batch_size = 100\n",
    "total_epoch = 30\n",
    "learning_rate = 0.05\n",
    "use_cuda = torch.cuda.is_available() # [Question 1] What is 'torch.cuda.is_available' used for?\n",
    "\n",
    "model = p1CNN()\n",
    "\n",
    "if use_cuda:\n",
    "    model = model.cuda()\n",
    "# model.apply(weights_init)\n",
    "model.load_state_dict(tmp.state_dict())\n",
    "\n",
    "optimizer = optim.SGD(model.parameters(), lr=learning_rate) # [Question 5] What is 'optim.SGD' used for?\n",
    "criterion = nn.CrossEntropyLoss() # [Question 6] What is 'nn.CrossEntropyLoss' used for?\n",
    "lr_scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=3, gamma=0.5)\n",
    "\n",
    "\n",
    "para_init = model.state_dict()\n",
    "optim_init = optimizer.state_dict()\n",
    "torch.save(model.state_dict(), 'model_init.pt')\n",
    "torch.save(optimizer.state_dict(), 'optim_init.pt')\n",
    "print(model.state_dict())\n",
    "print(optim_init)\n",
    "\n",
    "# Batch size increasing.\n",
    "\n",
    "# model.load_state_dict(torch.load('model_init.pt'))\n",
    "# optimizer.load_state_dict(torch.load('optim_init.pt'))\n",
    "\n",
    "# model.load_state_dict(para_init)\n",
    "# optimizer.load_state_dict(optim_init)\n",
    "print(model.state_dict())\n",
    "bsi_train_loss = []\n",
    "bsi_test_acc = []\n",
    "for epoch in range(total_epoch):\n",
    "    if epoch < 3:\n",
    "        batch_size = 50\n",
    "    elif epoch < 6:\n",
    "        batch_size = 100\n",
    "    elif epoch < 9:\n",
    "        batch_size = 200\n",
    "    else:\n",
    "        batch_size = 400\n",
    "    train_loader = torch.utils.data.DataLoader(  # [Question 2] What is 'torch.utils.data.DataLoader' used for?\n",
    "                 dataset=train_set,\n",
    "                 batch_size=batch_size,\n",
    "                 shuffle=True)\n",
    "    test_loader = torch.utils.data.DataLoader(\n",
    "                dataset=test_set,\n",
    "                batch_size=batch_size,\n",
    "                shuffle=False)\n",
    "    # trainning\n",
    "    total_loss = 0\n",
    "    total_batch = 0\n",
    "    for batch_idx, (x, target) in enumerate(train_loader):\n",
    "        model.train()\n",
    "        optimizer.zero_grad() # [Question 7] What is 'optimizer.zero_grad()' used for?\n",
    "        if use_cuda:\n",
    "            x, target = x.cuda(), target.cuda()\n",
    "            \n",
    "        out = model(x)\n",
    "        loss = criterion(out, target) # [Question 8] What are 'out' and 'target' and what is output of criterion(out, target)?\n",
    "\n",
    "        total_loss += loss.item()\n",
    "        total_batch += 1\n",
    "        loss.backward() # [Question 9] What is 'loss.backward()' used for?\n",
    "        optimizer.step() # [Question 10] What is 'optimizer.step()' used for?\n",
    "    print ('==>>> epoch: {}, batch index: {}, train loss: {:.6f}'\n",
    "            .format(epoch, batch_idx+1, total_loss / total_batch))\n",
    "    bsi_train_loss.append(total_loss / total_batch)\n",
    "    \n",
    "    # testing\n",
    "    total_loss = 0\n",
    "    total_batch = 0\n",
    "    correct_cnt = 0\n",
    "    total_cnt = 0\n",
    "    \n",
    "    for batch_idx, (x, target) in enumerate(test_loader):\n",
    "        model.eval()\n",
    "        if use_cuda:\n",
    "            x, target = x.cuda(), target.cuda()\n",
    "            \n",
    "        out = model(x)\n",
    "        loss = criterion(out, target)\n",
    "        _, pred_label = torch.max(out.data, 1)\n",
    "        total_cnt += x.data.size()[0]\n",
    "        correct_cnt += (pred_label == target.data).sum().item()\n",
    "        \n",
    "        total_loss += loss.item()\n",
    "        total_batch += 1\n",
    "        \n",
    "    print ('==>>> epoch: {}, batch index: {}, test loss: {:.6f}, acc: {:.3f}'\n",
    "            .format(epoch, batch_idx+1, total_loss / total_batch, correct_cnt * 1.0 / total_cnt))\n",
    "    bsi_test_acc.append(correct_cnt * 1.0 / total_cnt)\n",
    "    # lr_scheduler.step()\n",
    "    for param_group in optimizer.param_groups:\n",
    "        print(param_group['lr'])\n",
    "    print(batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "output_type": "execute_result",
     "data": {
      "text/plain": "Text(0.5, 1.0, 'Comparison of test acc by \\nLR decay and Batch size increasing')"
     },
     "metadata": {},
     "execution_count": 129
    },
    {
     "output_type": "display_data",
     "data": {
      "text/plain": "<Figure size 1120x480 with 2 Axes>",
      "image/svg+xml": "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"no\"?>\r\n<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\r\n  \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\r\n<!-- Created with matplotlib (https://matplotlib.org/) -->\r\n<svg height=\"399.47175pt\" version=\"1.1\" viewBox=\"0 0 833.534631 399.47175\" width=\"833.534631pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\r\n <defs>\r\n  <style type=\"text/css\">\r\n*{stroke-linecap:butt;stroke-linejoin:round;}\r\n  </style>\r\n </defs>\r\n <g id=\"figure_1\">\r\n  <g id=\"patch_1\">\r\n   <path d=\"M 0 399.47175 \r\nL 833.534631 399.47175 \r\nL 833.534631 0 \r\nL 0 0 \r\nz\r\n\" style=\"fill:#ffffff;\"/>\r\n  </g>\r\n  <g id=\"axes_1\">\r\n   <g id=\"patch_2\">\r\n    <path d=\"M 43.78125 361.9155 \r\nL 398.872159 361.9155 \r\nL 398.872159 35.7555 \r\nL 43.78125 35.7555 \r\nz\r\n\" style=\"fill:#ffffff;\"/>\r\n   </g>\r\n   <g id=\"matplotlib.axis_1\">\r\n    <g id=\"xtick_1\">\r\n     <g id=\"line2d_1\">\r\n      <defs>\r\n       <path d=\"M 0 0 \r\nL 0 3.5 \r\n\" id=\"mb57ca4ec09\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n      </defs>\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"59.921746\" xlink:href=\"#mb57ca4ec09\" y=\"361.9155\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_1\">\r\n      <!-- 0 -->\r\n      <defs>\r\n       <path d=\"M 31.78125 66.40625 \r\nQ 24.171875 66.40625 20.328125 58.90625 \r\nQ 16.5 51.421875 16.5 36.375 \r\nQ 16.5 21.390625 20.328125 13.890625 \r\nQ 24.171875 6.390625 31.78125 6.390625 \r\nQ 39.453125 6.390625 43.28125 13.890625 \r\nQ 47.125 21.390625 47.125 36.375 \r\nQ 47.125 51.421875 43.28125 58.90625 \r\nQ 39.453125 66.40625 31.78125 66.40625 \r\nz\r\nM 31.78125 74.21875 \r\nQ 44.046875 74.21875 50.515625 64.515625 \r\nQ 56.984375 54.828125 56.984375 36.375 \r\nQ 56.984375 17.96875 50.515625 8.265625 \r\nQ 44.046875 -1.421875 31.78125 -1.421875 \r\nQ 19.53125 -1.421875 13.0625 8.265625 \r\nQ 6.59375 17.96875 6.59375 36.375 \r\nQ 6.59375 54.828125 13.0625 64.515625 \r\nQ 19.53125 74.21875 31.78125 74.21875 \r\nz\r\n\" id=\"DejaVuSans-48\"/>\r\n      </defs>\r\n      <g transform=\"translate(56.740496 376.513937)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_2\">\r\n     <g id=\"line2d_2\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"115.578628\" xlink:href=\"#mb57ca4ec09\" y=\"361.9155\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_2\">\r\n      <!-- 5 -->\r\n      <defs>\r\n       <path d=\"M 10.796875 72.90625 \r\nL 49.515625 72.90625 \r\nL 49.515625 64.59375 \r\nL 19.828125 64.59375 \r\nL 19.828125 46.734375 \r\nQ 21.96875 47.46875 24.109375 47.828125 \r\nQ 26.265625 48.1875 28.421875 48.1875 \r\nQ 40.625 48.1875 47.75 41.5 \r\nQ 54.890625 34.8125 54.890625 23.390625 \r\nQ 54.890625 11.625 47.5625 5.09375 \r\nQ 40.234375 -1.421875 26.90625 -1.421875 \r\nQ 22.3125 -1.421875 17.546875 -0.640625 \r\nQ 12.796875 0.140625 7.71875 1.703125 \r\nL 7.71875 11.625 \r\nQ 12.109375 9.234375 16.796875 8.0625 \r\nQ 21.484375 6.890625 26.703125 6.890625 \r\nQ 35.15625 6.890625 40.078125 11.328125 \r\nQ 45.015625 15.765625 45.015625 23.390625 \r\nQ 45.015625 31 40.078125 35.4375 \r\nQ 35.15625 39.890625 26.703125 39.890625 \r\nQ 22.75 39.890625 18.8125 39.015625 \r\nQ 14.890625 38.140625 10.796875 36.28125 \r\nz\r\n\" id=\"DejaVuSans-53\"/>\r\n      </defs>\r\n      <g transform=\"translate(112.397378 376.513937)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-53\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_3\">\r\n     <g id=\"line2d_3\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"171.23551\" xlink:href=\"#mb57ca4ec09\" y=\"361.9155\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_3\">\r\n      <!-- 10 -->\r\n      <defs>\r\n       <path d=\"M 12.40625 8.296875 \r\nL 28.515625 8.296875 \r\nL 28.515625 63.921875 \r\nL 10.984375 60.40625 \r\nL 10.984375 69.390625 \r\nL 28.421875 72.90625 \r\nL 38.28125 72.90625 \r\nL 38.28125 8.296875 \r\nL 54.390625 8.296875 \r\nL 54.390625 0 \r\nL 12.40625 0 \r\nz\r\n\" id=\"DejaVuSans-49\"/>\r\n      </defs>\r\n      <g transform=\"translate(164.87301 376.513937)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-49\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_4\">\r\n     <g id=\"line2d_4\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"226.892393\" xlink:href=\"#mb57ca4ec09\" y=\"361.9155\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_4\">\r\n      <!-- 15 -->\r\n      <g transform=\"translate(220.529893 376.513937)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-49\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_5\">\r\n     <g id=\"line2d_5\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"282.549275\" xlink:href=\"#mb57ca4ec09\" y=\"361.9155\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_5\">\r\n      <!-- 20 -->\r\n      <defs>\r\n       <path d=\"M 19.1875 8.296875 \r\nL 53.609375 8.296875 \r\nL 53.609375 0 \r\nL 7.328125 0 \r\nL 7.328125 8.296875 \r\nQ 12.9375 14.109375 22.625 23.890625 \r\nQ 32.328125 33.6875 34.8125 36.53125 \r\nQ 39.546875 41.84375 41.421875 45.53125 \r\nQ 43.3125 49.21875 43.3125 52.78125 \r\nQ 43.3125 58.59375 39.234375 62.25 \r\nQ 35.15625 65.921875 28.609375 65.921875 \r\nQ 23.96875 65.921875 18.8125 64.3125 \r\nQ 13.671875 62.703125 7.8125 59.421875 \r\nL 7.8125 69.390625 \r\nQ 13.765625 71.78125 18.9375 73 \r\nQ 24.125 74.21875 28.421875 74.21875 \r\nQ 39.75 74.21875 46.484375 68.546875 \r\nQ 53.21875 62.890625 53.21875 53.421875 \r\nQ 53.21875 48.921875 51.53125 44.890625 \r\nQ 49.859375 40.875 45.40625 35.40625 \r\nQ 44.1875 33.984375 37.640625 27.21875 \r\nQ 31.109375 20.453125 19.1875 8.296875 \r\nz\r\n\" id=\"DejaVuSans-50\"/>\r\n      </defs>\r\n      <g transform=\"translate(276.186775 376.513937)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-50\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_6\">\r\n     <g id=\"line2d_6\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"338.206157\" xlink:href=\"#mb57ca4ec09\" y=\"361.9155\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_6\">\r\n      <!-- 25 -->\r\n      <g transform=\"translate(331.843657 376.513937)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-50\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_7\">\r\n     <g id=\"line2d_7\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"393.86304\" xlink:href=\"#mb57ca4ec09\" y=\"361.9155\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_7\">\r\n      <!-- 30 -->\r\n      <defs>\r\n       <path d=\"M 40.578125 39.3125 \r\nQ 47.65625 37.796875 51.625 33 \r\nQ 55.609375 28.21875 55.609375 21.1875 \r\nQ 55.609375 10.40625 48.1875 4.484375 \r\nQ 40.765625 -1.421875 27.09375 -1.421875 \r\nQ 22.515625 -1.421875 17.65625 -0.515625 \r\nQ 12.796875 0.390625 7.625 2.203125 \r\nL 7.625 11.71875 \r\nQ 11.71875 9.328125 16.59375 8.109375 \r\nQ 21.484375 6.890625 26.8125 6.890625 \r\nQ 36.078125 6.890625 40.9375 10.546875 \r\nQ 45.796875 14.203125 45.796875 21.1875 \r\nQ 45.796875 27.640625 41.28125 31.265625 \r\nQ 36.765625 34.90625 28.71875 34.90625 \r\nL 20.21875 34.90625 \r\nL 20.21875 43.015625 \r\nL 29.109375 43.015625 \r\nQ 36.375 43.015625 40.234375 45.921875 \r\nQ 44.09375 48.828125 44.09375 54.296875 \r\nQ 44.09375 59.90625 40.109375 62.90625 \r\nQ 36.140625 65.921875 28.71875 65.921875 \r\nQ 24.65625 65.921875 20.015625 65.03125 \r\nQ 15.375 64.15625 9.8125 62.3125 \r\nL 9.8125 71.09375 \r\nQ 15.4375 72.65625 20.34375 73.4375 \r\nQ 25.25 74.21875 29.59375 74.21875 \r\nQ 40.828125 74.21875 47.359375 69.109375 \r\nQ 53.90625 64.015625 53.90625 55.328125 \r\nQ 53.90625 49.265625 50.4375 45.09375 \r\nQ 46.96875 40.921875 40.578125 39.3125 \r\nz\r\n\" id=\"DejaVuSans-51\"/>\r\n      </defs>\r\n      <g transform=\"translate(387.50054 376.513937)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-51\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"text_8\">\r\n     <!-- epoch -->\r\n     <defs>\r\n      <path d=\"M 56.203125 29.59375 \r\nL 56.203125 25.203125 \r\nL 14.890625 25.203125 \r\nQ 15.484375 15.921875 20.484375 11.0625 \r\nQ 25.484375 6.203125 34.421875 6.203125 \r\nQ 39.59375 6.203125 44.453125 7.46875 \r\nQ 49.3125 8.734375 54.109375 11.28125 \r\nL 54.109375 2.78125 \r\nQ 49.265625 0.734375 44.1875 -0.34375 \r\nQ 39.109375 -1.421875 33.890625 -1.421875 \r\nQ 20.796875 -1.421875 13.15625 6.1875 \r\nQ 5.515625 13.8125 5.515625 26.8125 \r\nQ 5.515625 40.234375 12.765625 48.109375 \r\nQ 20.015625 56 32.328125 56 \r\nQ 43.359375 56 49.78125 48.890625 \r\nQ 56.203125 41.796875 56.203125 29.59375 \r\nz\r\nM 47.21875 32.234375 \r\nQ 47.125 39.59375 43.09375 43.984375 \r\nQ 39.0625 48.390625 32.421875 48.390625 \r\nQ 24.90625 48.390625 20.390625 44.140625 \r\nQ 15.875 39.890625 15.1875 32.171875 \r\nz\r\n\" id=\"DejaVuSans-101\"/>\r\n      <path d=\"M 18.109375 8.203125 \r\nL 18.109375 -20.796875 \r\nL 9.078125 -20.796875 \r\nL 9.078125 54.6875 \r\nL 18.109375 54.6875 \r\nL 18.109375 46.390625 \r\nQ 20.953125 51.265625 25.265625 53.625 \r\nQ 29.59375 56 35.59375 56 \r\nQ 45.5625 56 51.78125 48.09375 \r\nQ 58.015625 40.1875 58.015625 27.296875 \r\nQ 58.015625 14.40625 51.78125 6.484375 \r\nQ 45.5625 -1.421875 35.59375 -1.421875 \r\nQ 29.59375 -1.421875 25.265625 0.953125 \r\nQ 20.953125 3.328125 18.109375 8.203125 \r\nz\r\nM 48.6875 27.296875 \r\nQ 48.6875 37.203125 44.609375 42.84375 \r\nQ 40.53125 48.484375 33.40625 48.484375 \r\nQ 26.265625 48.484375 22.1875 42.84375 \r\nQ 18.109375 37.203125 18.109375 27.296875 \r\nQ 18.109375 17.390625 22.1875 11.75 \r\nQ 26.265625 6.109375 33.40625 6.109375 \r\nQ 40.53125 6.109375 44.609375 11.75 \r\nQ 48.6875 17.390625 48.6875 27.296875 \r\nz\r\n\" id=\"DejaVuSans-112\"/>\r\n      <path d=\"M 30.609375 48.390625 \r\nQ 23.390625 48.390625 19.1875 42.75 \r\nQ 14.984375 37.109375 14.984375 27.296875 \r\nQ 14.984375 17.484375 19.15625 11.84375 \r\nQ 23.34375 6.203125 30.609375 6.203125 \r\nQ 37.796875 6.203125 41.984375 11.859375 \r\nQ 46.1875 17.53125 46.1875 27.296875 \r\nQ 46.1875 37.015625 41.984375 42.703125 \r\nQ 37.796875 48.390625 30.609375 48.390625 \r\nz\r\nM 30.609375 56 \r\nQ 42.328125 56 49.015625 48.375 \r\nQ 55.71875 40.765625 55.71875 27.296875 \r\nQ 55.71875 13.875 49.015625 6.21875 \r\nQ 42.328125 -1.421875 30.609375 -1.421875 \r\nQ 18.84375 -1.421875 12.171875 6.21875 \r\nQ 5.515625 13.875 5.515625 27.296875 \r\nQ 5.515625 40.765625 12.171875 48.375 \r\nQ 18.84375 56 30.609375 56 \r\nz\r\n\" id=\"DejaVuSans-111\"/>\r\n      <path d=\"M 48.78125 52.59375 \r\nL 48.78125 44.1875 \r\nQ 44.96875 46.296875 41.140625 47.34375 \r\nQ 37.3125 48.390625 33.40625 48.390625 \r\nQ 24.65625 48.390625 19.8125 42.84375 \r\nQ 14.984375 37.3125 14.984375 27.296875 \r\nQ 14.984375 17.28125 19.8125 11.734375 \r\nQ 24.65625 6.203125 33.40625 6.203125 \r\nQ 37.3125 6.203125 41.140625 7.25 \r\nQ 44.96875 8.296875 48.78125 10.40625 \r\nL 48.78125 2.09375 \r\nQ 45.015625 0.34375 40.984375 -0.53125 \r\nQ 36.96875 -1.421875 32.421875 -1.421875 \r\nQ 20.0625 -1.421875 12.78125 6.34375 \r\nQ 5.515625 14.109375 5.515625 27.296875 \r\nQ 5.515625 40.671875 12.859375 48.328125 \r\nQ 20.21875 56 33.015625 56 \r\nQ 37.15625 56 41.109375 55.140625 \r\nQ 45.0625 54.296875 48.78125 52.59375 \r\nz\r\n\" id=\"DejaVuSans-99\"/>\r\n      <path d=\"M 54.890625 33.015625 \r\nL 54.890625 0 \r\nL 45.90625 0 \r\nL 45.90625 32.71875 \r\nQ 45.90625 40.484375 42.875 44.328125 \r\nQ 39.84375 48.1875 33.796875 48.1875 \r\nQ 26.515625 48.1875 22.3125 43.546875 \r\nQ 18.109375 38.921875 18.109375 30.90625 \r\nL 18.109375 0 \r\nL 9.078125 0 \r\nL 9.078125 75.984375 \r\nL 18.109375 75.984375 \r\nL 18.109375 46.1875 \r\nQ 21.34375 51.125 25.703125 53.5625 \r\nQ 30.078125 56 35.796875 56 \r\nQ 45.21875 56 50.046875 50.171875 \r\nQ 54.890625 44.34375 54.890625 33.015625 \r\nz\r\n\" id=\"DejaVuSans-104\"/>\r\n     </defs>\r\n     <g transform=\"translate(206.09858 390.192062)scale(0.1 -0.1)\">\r\n      <use xlink:href=\"#DejaVuSans-101\"/>\r\n      <use x=\"61.523438\" xlink:href=\"#DejaVuSans-112\"/>\r\n      <use x=\"125\" xlink:href=\"#DejaVuSans-111\"/>\r\n      <use x=\"186.181641\" xlink:href=\"#DejaVuSans-99\"/>\r\n      <use x=\"241.162109\" xlink:href=\"#DejaVuSans-104\"/>\r\n     </g>\r\n    </g>\r\n   </g>\r\n   <g id=\"matplotlib.axis_2\">\r\n    <g id=\"ytick_1\">\r\n     <g id=\"line2d_8\">\r\n      <defs>\r\n       <path d=\"M 0 0 \r\nL -3.5 0 \r\n\" id=\"m73aa8f3890\" style=\"stroke:#000000;stroke-width:0.8;\"/>\r\n      </defs>\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m73aa8f3890\" y=\"324.475614\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_9\">\r\n      <!-- 0.4 -->\r\n      <defs>\r\n       <path d=\"M 10.6875 12.40625 \r\nL 21 12.40625 \r\nL 21 0 \r\nL 10.6875 0 \r\nz\r\n\" id=\"DejaVuSans-46\"/>\r\n       <path d=\"M 37.796875 64.3125 \r\nL 12.890625 25.390625 \r\nL 37.796875 25.390625 \r\nz\r\nM 35.203125 72.90625 \r\nL 47.609375 72.90625 \r\nL 47.609375 25.390625 \r\nL 58.015625 25.390625 \r\nL 58.015625 17.1875 \r\nL 47.609375 17.1875 \r\nL 47.609375 0 \r\nL 37.796875 0 \r\nL 37.796875 17.1875 \r\nL 4.890625 17.1875 \r\nL 4.890625 26.703125 \r\nz\r\n\" id=\"DejaVuSans-52\"/>\r\n      </defs>\r\n      <g transform=\"translate(20.878125 328.274833)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-52\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_2\">\r\n     <g id=\"line2d_9\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m73aa8f3890\" y=\"268.99667\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_10\">\r\n      <!-- 0.6 -->\r\n      <defs>\r\n       <path d=\"M 33.015625 40.375 \r\nQ 26.375 40.375 22.484375 35.828125 \r\nQ 18.609375 31.296875 18.609375 23.390625 \r\nQ 18.609375 15.53125 22.484375 10.953125 \r\nQ 26.375 6.390625 33.015625 6.390625 \r\nQ 39.65625 6.390625 43.53125 10.953125 \r\nQ 47.40625 15.53125 47.40625 23.390625 \r\nQ 47.40625 31.296875 43.53125 35.828125 \r\nQ 39.65625 40.375 33.015625 40.375 \r\nz\r\nM 52.59375 71.296875 \r\nL 52.59375 62.3125 \r\nQ 48.875 64.0625 45.09375 64.984375 \r\nQ 41.3125 65.921875 37.59375 65.921875 \r\nQ 27.828125 65.921875 22.671875 59.328125 \r\nQ 17.53125 52.734375 16.796875 39.40625 \r\nQ 19.671875 43.65625 24.015625 45.921875 \r\nQ 28.375 48.1875 33.59375 48.1875 \r\nQ 44.578125 48.1875 50.953125 41.515625 \r\nQ 57.328125 34.859375 57.328125 23.390625 \r\nQ 57.328125 12.15625 50.6875 5.359375 \r\nQ 44.046875 -1.421875 33.015625 -1.421875 \r\nQ 20.359375 -1.421875 13.671875 8.265625 \r\nQ 6.984375 17.96875 6.984375 36.375 \r\nQ 6.984375 53.65625 15.1875 63.9375 \r\nQ 23.390625 74.21875 37.203125 74.21875 \r\nQ 40.921875 74.21875 44.703125 73.484375 \r\nQ 48.484375 72.75 52.59375 71.296875 \r\nz\r\n\" id=\"DejaVuSans-54\"/>\r\n      </defs>\r\n      <g transform=\"translate(20.878125 272.795889)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-54\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_3\">\r\n     <g id=\"line2d_10\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m73aa8f3890\" y=\"213.517726\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_11\">\r\n      <!-- 0.8 -->\r\n      <defs>\r\n       <path d=\"M 31.78125 34.625 \r\nQ 24.75 34.625 20.71875 30.859375 \r\nQ 16.703125 27.09375 16.703125 20.515625 \r\nQ 16.703125 13.921875 20.71875 10.15625 \r\nQ 24.75 6.390625 31.78125 6.390625 \r\nQ 38.8125 6.390625 42.859375 10.171875 \r\nQ 46.921875 13.96875 46.921875 20.515625 \r\nQ 46.921875 27.09375 42.890625 30.859375 \r\nQ 38.875 34.625 31.78125 34.625 \r\nz\r\nM 21.921875 38.8125 \r\nQ 15.578125 40.375 12.03125 44.71875 \r\nQ 8.5 49.078125 8.5 55.328125 \r\nQ 8.5 64.0625 14.71875 69.140625 \r\nQ 20.953125 74.21875 31.78125 74.21875 \r\nQ 42.671875 74.21875 48.875 69.140625 \r\nQ 55.078125 64.0625 55.078125 55.328125 \r\nQ 55.078125 49.078125 51.53125 44.71875 \r\nQ 48 40.375 41.703125 38.8125 \r\nQ 48.828125 37.15625 52.796875 32.3125 \r\nQ 56.78125 27.484375 56.78125 20.515625 \r\nQ 56.78125 9.90625 50.3125 4.234375 \r\nQ 43.84375 -1.421875 31.78125 -1.421875 \r\nQ 19.734375 -1.421875 13.25 4.234375 \r\nQ 6.78125 9.90625 6.78125 20.515625 \r\nQ 6.78125 27.484375 10.78125 32.3125 \r\nQ 14.796875 37.15625 21.921875 38.8125 \r\nz\r\nM 18.3125 54.390625 \r\nQ 18.3125 48.734375 21.84375 45.5625 \r\nQ 25.390625 42.390625 31.78125 42.390625 \r\nQ 38.140625 42.390625 41.71875 45.5625 \r\nQ 45.3125 48.734375 45.3125 54.390625 \r\nQ 45.3125 60.0625 41.71875 63.234375 \r\nQ 38.140625 66.40625 31.78125 66.40625 \r\nQ 25.390625 66.40625 21.84375 63.234375 \r\nQ 18.3125 60.0625 18.3125 54.390625 \r\nz\r\n\" id=\"DejaVuSans-56\"/>\r\n      </defs>\r\n      <g transform=\"translate(20.878125 217.316944)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-56\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_4\">\r\n     <g id=\"line2d_11\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m73aa8f3890\" y=\"158.038781\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_12\">\r\n      <!-- 1.0 -->\r\n      <g transform=\"translate(20.878125 161.838)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-49\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_5\">\r\n     <g id=\"line2d_12\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m73aa8f3890\" y=\"102.559837\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_13\">\r\n      <!-- 1.2 -->\r\n      <g transform=\"translate(20.878125 106.359056)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-49\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-50\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_6\">\r\n     <g id=\"line2d_13\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"43.78125\" xlink:href=\"#m73aa8f3890\" y=\"47.080893\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_14\">\r\n      <!-- 1.4 -->\r\n      <g transform=\"translate(20.878125 50.880111)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-49\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-52\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"text_15\">\r\n     <!-- Loss -->\r\n     <defs>\r\n      <path d=\"M 9.8125 72.90625 \r\nL 19.671875 72.90625 \r\nL 19.671875 8.296875 \r\nL 55.171875 8.296875 \r\nL 55.171875 0 \r\nL 9.8125 0 \r\nz\r\n\" id=\"DejaVuSans-76\"/>\r\n      <path d=\"M 44.28125 53.078125 \r\nL 44.28125 44.578125 \r\nQ 40.484375 46.53125 36.375 47.5 \r\nQ 32.28125 48.484375 27.875 48.484375 \r\nQ 21.1875 48.484375 17.84375 46.4375 \r\nQ 14.5 44.390625 14.5 40.28125 \r\nQ 14.5 37.15625 16.890625 35.375 \r\nQ 19.28125 33.59375 26.515625 31.984375 \r\nL 29.59375 31.296875 \r\nQ 39.15625 29.25 43.1875 25.515625 \r\nQ 47.21875 21.78125 47.21875 15.09375 \r\nQ 47.21875 7.46875 41.1875 3.015625 \r\nQ 35.15625 -1.421875 24.609375 -1.421875 \r\nQ 20.21875 -1.421875 15.453125 -0.5625 \r\nQ 10.6875 0.296875 5.421875 2 \r\nL 5.421875 11.28125 \r\nQ 10.40625 8.6875 15.234375 7.390625 \r\nQ 20.0625 6.109375 24.8125 6.109375 \r\nQ 31.15625 6.109375 34.5625 8.28125 \r\nQ 37.984375 10.453125 37.984375 14.40625 \r\nQ 37.984375 18.0625 35.515625 20.015625 \r\nQ 33.0625 21.96875 24.703125 23.78125 \r\nL 21.578125 24.515625 \r\nQ 13.234375 26.265625 9.515625 29.90625 \r\nQ 5.8125 33.546875 5.8125 39.890625 \r\nQ 5.8125 47.609375 11.28125 51.796875 \r\nQ 16.75 56 26.8125 56 \r\nQ 31.78125 56 36.171875 55.265625 \r\nQ 40.578125 54.546875 44.28125 53.078125 \r\nz\r\n\" id=\"DejaVuSans-115\"/>\r\n     </defs>\r\n     <g transform=\"translate(14.798438 209.889406)rotate(-90)scale(0.1 -0.1)\">\r\n      <use xlink:href=\"#DejaVuSans-76\"/>\r\n      <use x=\"55.697266\" xlink:href=\"#DejaVuSans-111\"/>\r\n      <use x=\"116.878906\" xlink:href=\"#DejaVuSans-115\"/>\r\n      <use x=\"168.978516\" xlink:href=\"#DejaVuSans-115\"/>\r\n     </g>\r\n    </g>\r\n   </g>\r\n   <g id=\"line2d_14\">\r\n    <path clip-path=\"url(#p4f5a07b44d)\" d=\"M 59.921746 56.517369 \r\nL 71.053122 303.802949 \r\nL 82.184499 315.943332 \r\nL 93.315875 325.496699 \r\nL 104.447252 327.907202 \r\nL 115.578628 329.787146 \r\nL 126.710005 334.39236 \r\nL 137.841381 334.929387 \r\nL 148.972758 335.654259 \r\nL 160.104134 337.894182 \r\nL 171.23551 338.474827 \r\nL 182.366887 338.932329 \r\nL 193.498263 339.390097 \r\nL 204.62964 339.630148 \r\nL 215.761016 340.036643 \r\nL 226.892393 340.120495 \r\nL 238.023769 341.070451 \r\nL 249.155146 341.478195 \r\nL 260.286522 342.009414 \r\nL 271.417899 341.842837 \r\nL 282.549275 342.005314 \r\nL 293.680652 343.264692 \r\nL 304.812028 343.645981 \r\nL 315.943404 344.019774 \r\nL 327.074781 344.464743 \r\nL 338.206157 345.206041 \r\nL 349.337534 345.244932 \r\nL 360.46891 346.646761 \r\nL 371.600287 345.456164 \r\nL 382.731663 347.090045 \r\n\" style=\"fill:none;stroke:#1f77b4;stroke-linecap:square;stroke-width:1.5;\"/>\r\n   </g>\r\n   <g id=\"line2d_15\">\r\n    <path clip-path=\"url(#p4f5a07b44d)\" d=\"M 59.921746 50.580955 \r\nL 71.053122 303.878892 \r\nL 82.184499 316.592811 \r\nL 93.315875 327.956874 \r\nL 104.447252 330.607435 \r\nL 115.578628 332.995914 \r\nL 126.710005 336.812361 \r\nL 137.841381 338.101687 \r\nL 148.972758 339.320189 \r\nL 160.104134 341.252861 \r\nL 171.23551 341.825541 \r\nL 182.366887 342.545358 \r\nL 193.498263 343.454425 \r\nL 204.62964 343.824614 \r\nL 215.761016 344.141074 \r\nL 226.892393 344.617341 \r\nL 238.023769 344.810395 \r\nL 249.155146 344.958763 \r\nL 260.286522 345.254307 \r\nL 271.417899 345.316157 \r\nL 282.549275 345.410049 \r\nL 293.680652 345.524935 \r\nL 304.812028 345.571581 \r\nL 315.943404 345.600788 \r\nL 327.074781 345.676012 \r\nL 338.206157 345.69696 \r\nL 349.337534 345.716494 \r\nL 360.46891 345.751501 \r\nL 371.600287 345.763342 \r\nL 382.731663 345.771664 \r\n\" style=\"fill:none;stroke:#ff7f0e;stroke-linecap:square;stroke-width:1.5;\"/>\r\n   </g>\r\n   <g id=\"patch_3\">\r\n    <path d=\"M 43.78125 361.9155 \r\nL 43.78125 35.7555 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n   <g id=\"patch_4\">\r\n    <path d=\"M 398.872159 361.9155 \r\nL 398.872159 35.7555 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n   <g id=\"patch_5\">\r\n    <path d=\"M 43.78125 361.9155 \r\nL 398.872159 361.9155 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n   <g id=\"patch_6\">\r\n    <path d=\"M 43.78125 35.7555 \r\nL 398.872159 35.7555 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n   <g id=\"text_16\">\r\n    <!-- Comparison of Loss by  -->\r\n    <defs>\r\n     <path d=\"M 64.40625 67.28125 \r\nL 64.40625 56.890625 \r\nQ 59.421875 61.53125 53.78125 63.8125 \r\nQ 48.140625 66.109375 41.796875 66.109375 \r\nQ 29.296875 66.109375 22.65625 58.46875 \r\nQ 16.015625 50.828125 16.015625 36.375 \r\nQ 16.015625 21.96875 22.65625 14.328125 \r\nQ 29.296875 6.6875 41.796875 6.6875 \r\nQ 48.140625 6.6875 53.78125 8.984375 \r\nQ 59.421875 11.28125 64.40625 15.921875 \r\nL 64.40625 5.609375 \r\nQ 59.234375 2.09375 53.4375 0.328125 \r\nQ 47.65625 -1.421875 41.21875 -1.421875 \r\nQ 24.65625 -1.421875 15.125 8.703125 \r\nQ 5.609375 18.84375 5.609375 36.375 \r\nQ 5.609375 53.953125 15.125 64.078125 \r\nQ 24.65625 74.21875 41.21875 74.21875 \r\nQ 47.75 74.21875 53.53125 72.484375 \r\nQ 59.328125 70.75 64.40625 67.28125 \r\nz\r\n\" id=\"DejaVuSans-67\"/>\r\n     <path d=\"M 52 44.1875 \r\nQ 55.375 50.25 60.0625 53.125 \r\nQ 64.75 56 71.09375 56 \r\nQ 79.640625 56 84.28125 50.015625 \r\nQ 88.921875 44.046875 88.921875 33.015625 \r\nL 88.921875 0 \r\nL 79.890625 0 \r\nL 79.890625 32.71875 \r\nQ 79.890625 40.578125 77.09375 44.375 \r\nQ 74.3125 48.1875 68.609375 48.1875 \r\nQ 61.625 48.1875 57.5625 43.546875 \r\nQ 53.515625 38.921875 53.515625 30.90625 \r\nL 53.515625 0 \r\nL 44.484375 0 \r\nL 44.484375 32.71875 \r\nQ 44.484375 40.625 41.703125 44.40625 \r\nQ 38.921875 48.1875 33.109375 48.1875 \r\nQ 26.21875 48.1875 22.15625 43.53125 \r\nQ 18.109375 38.875 18.109375 30.90625 \r\nL 18.109375 0 \r\nL 9.078125 0 \r\nL 9.078125 54.6875 \r\nL 18.109375 54.6875 \r\nL 18.109375 46.1875 \r\nQ 21.1875 51.21875 25.484375 53.609375 \r\nQ 29.78125 56 35.6875 56 \r\nQ 41.65625 56 45.828125 52.96875 \r\nQ 50 49.953125 52 44.1875 \r\nz\r\n\" id=\"DejaVuSans-109\"/>\r\n     <path d=\"M 34.28125 27.484375 \r\nQ 23.390625 27.484375 19.1875 25 \r\nQ 14.984375 22.515625 14.984375 16.5 \r\nQ 14.984375 11.71875 18.140625 8.90625 \r\nQ 21.296875 6.109375 26.703125 6.109375 \r\nQ 34.1875 6.109375 38.703125 11.40625 \r\nQ 43.21875 16.703125 43.21875 25.484375 \r\nL 43.21875 27.484375 \r\nz\r\nM 52.203125 31.203125 \r\nL 52.203125 0 \r\nL 43.21875 0 \r\nL 43.21875 8.296875 \r\nQ 40.140625 3.328125 35.546875 0.953125 \r\nQ 30.953125 -1.421875 24.3125 -1.421875 \r\nQ 15.921875 -1.421875 10.953125 3.296875 \r\nQ 6 8.015625 6 15.921875 \r\nQ 6 25.140625 12.171875 29.828125 \r\nQ 18.359375 34.515625 30.609375 34.515625 \r\nL 43.21875 34.515625 \r\nL 43.21875 35.40625 \r\nQ 43.21875 41.609375 39.140625 45 \r\nQ 35.0625 48.390625 27.6875 48.390625 \r\nQ 23 48.390625 18.546875 47.265625 \r\nQ 14.109375 46.140625 10.015625 43.890625 \r\nL 10.015625 52.203125 \r\nQ 14.9375 54.109375 19.578125 55.046875 \r\nQ 24.21875 56 28.609375 56 \r\nQ 40.484375 56 46.34375 49.84375 \r\nQ 52.203125 43.703125 52.203125 31.203125 \r\nz\r\n\" id=\"DejaVuSans-97\"/>\r\n     <path d=\"M 41.109375 46.296875 \r\nQ 39.59375 47.171875 37.8125 47.578125 \r\nQ 36.03125 48 33.890625 48 \r\nQ 26.265625 48 22.1875 43.046875 \r\nQ 18.109375 38.09375 18.109375 28.8125 \r\nL 18.109375 0 \r\nL 9.078125 0 \r\nL 9.078125 54.6875 \r\nL 18.109375 54.6875 \r\nL 18.109375 46.1875 \r\nQ 20.953125 51.171875 25.484375 53.578125 \r\nQ 30.03125 56 36.53125 56 \r\nQ 37.453125 56 38.578125 55.875 \r\nQ 39.703125 55.765625 41.0625 55.515625 \r\nz\r\n\" id=\"DejaVuSans-114\"/>\r\n     <path d=\"M 9.421875 54.6875 \r\nL 18.40625 54.6875 \r\nL 18.40625 0 \r\nL 9.421875 0 \r\nz\r\nM 9.421875 75.984375 \r\nL 18.40625 75.984375 \r\nL 18.40625 64.59375 \r\nL 9.421875 64.59375 \r\nz\r\n\" id=\"DejaVuSans-105\"/>\r\n     <path d=\"M 54.890625 33.015625 \r\nL 54.890625 0 \r\nL 45.90625 0 \r\nL 45.90625 32.71875 \r\nQ 45.90625 40.484375 42.875 44.328125 \r\nQ 39.84375 48.1875 33.796875 48.1875 \r\nQ 26.515625 48.1875 22.3125 43.546875 \r\nQ 18.109375 38.921875 18.109375 30.90625 \r\nL 18.109375 0 \r\nL 9.078125 0 \r\nL 9.078125 54.6875 \r\nL 18.109375 54.6875 \r\nL 18.109375 46.1875 \r\nQ 21.34375 51.125 25.703125 53.5625 \r\nQ 30.078125 56 35.796875 56 \r\nQ 45.21875 56 50.046875 50.171875 \r\nQ 54.890625 44.34375 54.890625 33.015625 \r\nz\r\n\" id=\"DejaVuSans-110\"/>\r\n     <path id=\"DejaVuSans-32\"/>\r\n     <path d=\"M 37.109375 75.984375 \r\nL 37.109375 68.5 \r\nL 28.515625 68.5 \r\nQ 23.6875 68.5 21.796875 66.546875 \r\nQ 19.921875 64.59375 19.921875 59.515625 \r\nL 19.921875 54.6875 \r\nL 34.71875 54.6875 \r\nL 34.71875 47.703125 \r\nL 19.921875 47.703125 \r\nL 19.921875 0 \r\nL 10.890625 0 \r\nL 10.890625 47.703125 \r\nL 2.296875 47.703125 \r\nL 2.296875 54.6875 \r\nL 10.890625 54.6875 \r\nL 10.890625 58.5 \r\nQ 10.890625 67.625 15.140625 71.796875 \r\nQ 19.390625 75.984375 28.609375 75.984375 \r\nz\r\n\" id=\"DejaVuSans-102\"/>\r\n     <path d=\"M 48.6875 27.296875 \r\nQ 48.6875 37.203125 44.609375 42.84375 \r\nQ 40.53125 48.484375 33.40625 48.484375 \r\nQ 26.265625 48.484375 22.1875 42.84375 \r\nQ 18.109375 37.203125 18.109375 27.296875 \r\nQ 18.109375 17.390625 22.1875 11.75 \r\nQ 26.265625 6.109375 33.40625 6.109375 \r\nQ 40.53125 6.109375 44.609375 11.75 \r\nQ 48.6875 17.390625 48.6875 27.296875 \r\nz\r\nM 18.109375 46.390625 \r\nQ 20.953125 51.265625 25.265625 53.625 \r\nQ 29.59375 56 35.59375 56 \r\nQ 45.5625 56 51.78125 48.09375 \r\nQ 58.015625 40.1875 58.015625 27.296875 \r\nQ 58.015625 14.40625 51.78125 6.484375 \r\nQ 45.5625 -1.421875 35.59375 -1.421875 \r\nQ 29.59375 -1.421875 25.265625 0.953125 \r\nQ 20.953125 3.328125 18.109375 8.203125 \r\nL 18.109375 0 \r\nL 9.078125 0 \r\nL 9.078125 75.984375 \r\nL 18.109375 75.984375 \r\nz\r\n\" id=\"DejaVuSans-98\"/>\r\n     <path d=\"M 32.171875 -5.078125 \r\nQ 28.375 -14.84375 24.75 -17.8125 \r\nQ 21.140625 -20.796875 15.09375 -20.796875 \r\nL 7.90625 -20.796875 \r\nL 7.90625 -13.28125 \r\nL 13.1875 -13.28125 \r\nQ 16.890625 -13.28125 18.9375 -11.515625 \r\nQ 21 -9.765625 23.484375 -3.21875 \r\nL 25.09375 0.875 \r\nL 2.984375 54.6875 \r\nL 12.5 54.6875 \r\nL 29.59375 11.921875 \r\nL 46.6875 54.6875 \r\nL 56.203125 54.6875 \r\nz\r\n\" id=\"DejaVuSans-121\"/>\r\n    </defs>\r\n    <g transform=\"translate(151.366705 16.318125)scale(0.12 -0.12)\">\r\n     <use xlink:href=\"#DejaVuSans-67\"/>\r\n     <use x=\"69.824219\" xlink:href=\"#DejaVuSans-111\"/>\r\n     <use x=\"131.005859\" xlink:href=\"#DejaVuSans-109\"/>\r\n     <use x=\"228.417969\" xlink:href=\"#DejaVuSans-112\"/>\r\n     <use x=\"291.894531\" xlink:href=\"#DejaVuSans-97\"/>\r\n     <use x=\"353.173828\" xlink:href=\"#DejaVuSans-114\"/>\r\n     <use x=\"394.287109\" xlink:href=\"#DejaVuSans-105\"/>\r\n     <use x=\"422.070312\" xlink:href=\"#DejaVuSans-115\"/>\r\n     <use x=\"474.169922\" xlink:href=\"#DejaVuSans-111\"/>\r\n     <use x=\"535.351562\" xlink:href=\"#DejaVuSans-110\"/>\r\n     <use x=\"598.730469\" xlink:href=\"#DejaVuSans-32\"/>\r\n     <use x=\"630.517578\" xlink:href=\"#DejaVuSans-111\"/>\r\n     <use x=\"691.699219\" xlink:href=\"#DejaVuSans-102\"/>\r\n     <use x=\"726.904297\" xlink:href=\"#DejaVuSans-32\"/>\r\n     <use x=\"758.691406\" xlink:href=\"#DejaVuSans-76\"/>\r\n     <use x=\"814.388672\" xlink:href=\"#DejaVuSans-111\"/>\r\n     <use x=\"875.570312\" xlink:href=\"#DejaVuSans-115\"/>\r\n     <use x=\"927.669922\" xlink:href=\"#DejaVuSans-115\"/>\r\n     <use x=\"979.769531\" xlink:href=\"#DejaVuSans-32\"/>\r\n     <use x=\"1011.556641\" xlink:href=\"#DejaVuSans-98\"/>\r\n     <use x=\"1075.033203\" xlink:href=\"#DejaVuSans-121\"/>\r\n     <use x=\"1134.212891\" xlink:href=\"#DejaVuSans-32\"/>\r\n    </g>\r\n    <!-- LR decay and Batch size increasing -->\r\n    <defs>\r\n     <path d=\"M 44.390625 34.1875 \r\nQ 47.5625 33.109375 50.5625 29.59375 \r\nQ 53.5625 26.078125 56.59375 19.921875 \r\nL 66.609375 0 \r\nL 56 0 \r\nL 46.6875 18.703125 \r\nQ 43.0625 26.03125 39.671875 28.421875 \r\nQ 36.28125 30.8125 30.421875 30.8125 \r\nL 19.671875 30.8125 \r\nL 19.671875 0 \r\nL 9.8125 0 \r\nL 9.8125 72.90625 \r\nL 32.078125 72.90625 \r\nQ 44.578125 72.90625 50.734375 67.671875 \r\nQ 56.890625 62.453125 56.890625 51.90625 \r\nQ 56.890625 45.015625 53.6875 40.46875 \r\nQ 50.484375 35.9375 44.390625 34.1875 \r\nz\r\nM 19.671875 64.796875 \r\nL 19.671875 38.921875 \r\nL 32.078125 38.921875 \r\nQ 39.203125 38.921875 42.84375 42.21875 \r\nQ 46.484375 45.515625 46.484375 51.90625 \r\nQ 46.484375 58.296875 42.84375 61.546875 \r\nQ 39.203125 64.796875 32.078125 64.796875 \r\nz\r\n\" id=\"DejaVuSans-82\"/>\r\n     <path d=\"M 45.40625 46.390625 \r\nL 45.40625 75.984375 \r\nL 54.390625 75.984375 \r\nL 54.390625 0 \r\nL 45.40625 0 \r\nL 45.40625 8.203125 \r\nQ 42.578125 3.328125 38.25 0.953125 \r\nQ 33.9375 -1.421875 27.875 -1.421875 \r\nQ 17.96875 -1.421875 11.734375 6.484375 \r\nQ 5.515625 14.40625 5.515625 27.296875 \r\nQ 5.515625 40.1875 11.734375 48.09375 \r\nQ 17.96875 56 27.875 56 \r\nQ 33.9375 56 38.25 53.625 \r\nQ 42.578125 51.265625 45.40625 46.390625 \r\nz\r\nM 14.796875 27.296875 \r\nQ 14.796875 17.390625 18.875 11.75 \r\nQ 22.953125 6.109375 30.078125 6.109375 \r\nQ 37.203125 6.109375 41.296875 11.75 \r\nQ 45.40625 17.390625 45.40625 27.296875 \r\nQ 45.40625 37.203125 41.296875 42.84375 \r\nQ 37.203125 48.484375 30.078125 48.484375 \r\nQ 22.953125 48.484375 18.875 42.84375 \r\nQ 14.796875 37.203125 14.796875 27.296875 \r\nz\r\n\" id=\"DejaVuSans-100\"/>\r\n     <path d=\"M 19.671875 34.8125 \r\nL 19.671875 8.109375 \r\nL 35.5 8.109375 \r\nQ 43.453125 8.109375 47.28125 11.40625 \r\nQ 51.125 14.703125 51.125 21.484375 \r\nQ 51.125 28.328125 47.28125 31.5625 \r\nQ 43.453125 34.8125 35.5 34.8125 \r\nz\r\nM 19.671875 64.796875 \r\nL 19.671875 42.828125 \r\nL 34.28125 42.828125 \r\nQ 41.5 42.828125 45.03125 45.53125 \r\nQ 48.578125 48.25 48.578125 53.8125 \r\nQ 48.578125 59.328125 45.03125 62.0625 \r\nQ 41.5 64.796875 34.28125 64.796875 \r\nz\r\nM 9.8125 72.90625 \r\nL 35.015625 72.90625 \r\nQ 46.296875 72.90625 52.390625 68.21875 \r\nQ 58.5 63.53125 58.5 54.890625 \r\nQ 58.5 48.1875 55.375 44.234375 \r\nQ 52.25 40.28125 46.1875 39.3125 \r\nQ 53.46875 37.75 57.5 32.78125 \r\nQ 61.53125 27.828125 61.53125 20.40625 \r\nQ 61.53125 10.640625 54.890625 5.3125 \r\nQ 48.25 0 35.984375 0 \r\nL 9.8125 0 \r\nz\r\n\" id=\"DejaVuSans-66\"/>\r\n     <path d=\"M 18.3125 70.21875 \r\nL 18.3125 54.6875 \r\nL 36.8125 54.6875 \r\nL 36.8125 47.703125 \r\nL 18.3125 47.703125 \r\nL 18.3125 18.015625 \r\nQ 18.3125 11.328125 20.140625 9.421875 \r\nQ 21.96875 7.515625 27.59375 7.515625 \r\nL 36.8125 7.515625 \r\nL 36.8125 0 \r\nL 27.59375 0 \r\nQ 17.1875 0 13.234375 3.875 \r\nQ 9.28125 7.765625 9.28125 18.015625 \r\nL 9.28125 47.703125 \r\nL 2.6875 47.703125 \r\nL 2.6875 54.6875 \r\nL 9.28125 54.6875 \r\nL 9.28125 70.21875 \r\nz\r\n\" id=\"DejaVuSans-116\"/>\r\n     <path d=\"M 5.515625 54.6875 \r\nL 48.1875 54.6875 \r\nL 48.1875 46.484375 \r\nL 14.40625 7.171875 \r\nL 48.1875 7.171875 \r\nL 48.1875 0 \r\nL 4.296875 0 \r\nL 4.296875 8.203125 \r\nL 38.09375 47.515625 \r\nL 5.515625 47.515625 \r\nz\r\n\" id=\"DejaVuSans-122\"/>\r\n     <path d=\"M 45.40625 27.984375 \r\nQ 45.40625 37.75 41.375 43.109375 \r\nQ 37.359375 48.484375 30.078125 48.484375 \r\nQ 22.859375 48.484375 18.828125 43.109375 \r\nQ 14.796875 37.75 14.796875 27.984375 \r\nQ 14.796875 18.265625 18.828125 12.890625 \r\nQ 22.859375 7.515625 30.078125 7.515625 \r\nQ 37.359375 7.515625 41.375 12.890625 \r\nQ 45.40625 18.265625 45.40625 27.984375 \r\nz\r\nM 54.390625 6.78125 \r\nQ 54.390625 -7.171875 48.1875 -13.984375 \r\nQ 42 -20.796875 29.203125 -20.796875 \r\nQ 24.46875 -20.796875 20.265625 -20.09375 \r\nQ 16.0625 -19.390625 12.109375 -17.921875 \r\nL 12.109375 -9.1875 \r\nQ 16.0625 -11.328125 19.921875 -12.34375 \r\nQ 23.78125 -13.375 27.78125 -13.375 \r\nQ 36.625 -13.375 41.015625 -8.765625 \r\nQ 45.40625 -4.15625 45.40625 5.171875 \r\nL 45.40625 9.625 \r\nQ 42.625 4.78125 38.28125 2.390625 \r\nQ 33.9375 0 27.875 0 \r\nQ 17.828125 0 11.671875 7.65625 \r\nQ 5.515625 15.328125 5.515625 27.984375 \r\nQ 5.515625 40.671875 11.671875 48.328125 \r\nQ 17.828125 56 27.875 56 \r\nQ 33.9375 56 38.28125 53.609375 \r\nQ 42.625 51.21875 45.40625 46.390625 \r\nL 45.40625 54.6875 \r\nL 54.390625 54.6875 \r\nz\r\n\" id=\"DejaVuSans-103\"/>\r\n    </defs>\r\n    <g transform=\"translate(115.077017 29.7555)scale(0.12 -0.12)\">\r\n     <use xlink:href=\"#DejaVuSans-76\"/>\r\n     <use x=\"55.712891\" xlink:href=\"#DejaVuSans-82\"/>\r\n     <use x=\"125.195312\" xlink:href=\"#DejaVuSans-32\"/>\r\n     <use x=\"156.982422\" xlink:href=\"#DejaVuSans-100\"/>\r\n     <use x=\"220.458984\" xlink:href=\"#DejaVuSans-101\"/>\r\n     <use x=\"281.982422\" xlink:href=\"#DejaVuSans-99\"/>\r\n     <use x=\"336.962891\" xlink:href=\"#DejaVuSans-97\"/>\r\n     <use x=\"398.242188\" xlink:href=\"#DejaVuSans-121\"/>\r\n     <use x=\"457.421875\" xlink:href=\"#DejaVuSans-32\"/>\r\n     <use x=\"489.208984\" xlink:href=\"#DejaVuSans-97\"/>\r\n     <use x=\"550.488281\" xlink:href=\"#DejaVuSans-110\"/>\r\n     <use x=\"613.867188\" xlink:href=\"#DejaVuSans-100\"/>\r\n     <use x=\"677.34375\" xlink:href=\"#DejaVuSans-32\"/>\r\n     <use x=\"709.130859\" xlink:href=\"#DejaVuSans-66\"/>\r\n     <use x=\"777.734375\" xlink:href=\"#DejaVuSans-97\"/>\r\n     <use x=\"839.013672\" xlink:href=\"#DejaVuSans-116\"/>\r\n     <use x=\"878.222656\" xlink:href=\"#DejaVuSans-99\"/>\r\n     <use x=\"933.203125\" xlink:href=\"#DejaVuSans-104\"/>\r\n     <use x=\"996.582031\" xlink:href=\"#DejaVuSans-32\"/>\r\n     <use x=\"1028.369141\" xlink:href=\"#DejaVuSans-115\"/>\r\n     <use x=\"1080.46875\" xlink:href=\"#DejaVuSans-105\"/>\r\n     <use x=\"1108.251953\" xlink:href=\"#DejaVuSans-122\"/>\r\n     <use x=\"1160.742188\" xlink:href=\"#DejaVuSans-101\"/>\r\n     <use x=\"1222.265625\" xlink:href=\"#DejaVuSans-32\"/>\r\n     <use x=\"1254.052734\" xlink:href=\"#DejaVuSans-105\"/>\r\n     <use x=\"1281.835938\" xlink:href=\"#DejaVuSans-110\"/>\r\n     <use x=\"1345.214844\" xlink:href=\"#DejaVuSans-99\"/>\r\n     <use x=\"1400.195312\" xlink:href=\"#DejaVuSans-114\"/>\r\n     <use x=\"1441.277344\" xlink:href=\"#DejaVuSans-101\"/>\r\n     <use x=\"1502.800781\" xlink:href=\"#DejaVuSans-97\"/>\r\n     <use x=\"1564.080078\" xlink:href=\"#DejaVuSans-115\"/>\r\n     <use x=\"1616.179688\" xlink:href=\"#DejaVuSans-105\"/>\r\n     <use x=\"1643.962891\" xlink:href=\"#DejaVuSans-110\"/>\r\n     <use x=\"1707.341797\" xlink:href=\"#DejaVuSans-103\"/>\r\n    </g>\r\n   </g>\r\n   <g id=\"legend_1\">\r\n    <g id=\"patch_7\">\r\n     <path d=\"M 259.394034 73.11175 \r\nL 391.872159 73.11175 \r\nQ 393.872159 73.11175 393.872159 71.11175 \r\nL 393.872159 42.7555 \r\nQ 393.872159 40.7555 391.872159 40.7555 \r\nL 259.394034 40.7555 \r\nQ 257.394034 40.7555 257.394034 42.7555 \r\nL 257.394034 71.11175 \r\nQ 257.394034 73.11175 259.394034 73.11175 \r\nz\r\n\" style=\"fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;\"/>\r\n    </g>\r\n    <g id=\"line2d_16\">\r\n     <path d=\"M 261.394034 48.853937 \r\nL 281.394034 48.853937 \r\n\" style=\"fill:none;stroke:#1f77b4;stroke-linecap:square;stroke-width:1.5;\"/>\r\n    </g>\r\n    <g id=\"line2d_17\"/>\r\n    <g id=\"text_17\">\r\n     <!-- Batch size increase -->\r\n     <g transform=\"translate(289.394034 52.353937)scale(0.1 -0.1)\">\r\n      <use xlink:href=\"#DejaVuSans-66\"/>\r\n      <use x=\"68.603516\" xlink:href=\"#DejaVuSans-97\"/>\r\n      <use x=\"129.882812\" xlink:href=\"#DejaVuSans-116\"/>\r\n      <use x=\"169.091797\" xlink:href=\"#DejaVuSans-99\"/>\r\n      <use x=\"224.072266\" xlink:href=\"#DejaVuSans-104\"/>\r\n      <use x=\"287.451172\" xlink:href=\"#DejaVuSans-32\"/>\r\n      <use x=\"319.238281\" xlink:href=\"#DejaVuSans-115\"/>\r\n      <use x=\"371.337891\" xlink:href=\"#DejaVuSans-105\"/>\r\n      <use x=\"399.121094\" xlink:href=\"#DejaVuSans-122\"/>\r\n      <use x=\"451.611328\" xlink:href=\"#DejaVuSans-101\"/>\r\n      <use x=\"513.134766\" xlink:href=\"#DejaVuSans-32\"/>\r\n      <use x=\"544.921875\" xlink:href=\"#DejaVuSans-105\"/>\r\n      <use x=\"572.705078\" xlink:href=\"#DejaVuSans-110\"/>\r\n      <use x=\"636.083984\" xlink:href=\"#DejaVuSans-99\"/>\r\n      <use x=\"691.064453\" xlink:href=\"#DejaVuSans-114\"/>\r\n      <use x=\"732.146484\" xlink:href=\"#DejaVuSans-101\"/>\r\n      <use x=\"793.669922\" xlink:href=\"#DejaVuSans-97\"/>\r\n      <use x=\"854.949219\" xlink:href=\"#DejaVuSans-115\"/>\r\n      <use x=\"907.048828\" xlink:href=\"#DejaVuSans-101\"/>\r\n     </g>\r\n    </g>\r\n    <g id=\"line2d_18\">\r\n     <path d=\"M 261.394034 63.532062 \r\nL 281.394034 63.532062 \r\n\" style=\"fill:none;stroke:#ff7f0e;stroke-linecap:square;stroke-width:1.5;\"/>\r\n    </g>\r\n    <g id=\"line2d_19\"/>\r\n    <g id=\"text_18\">\r\n     <!-- Learning rate decay -->\r\n     <g transform=\"translate(289.394034 67.032062)scale(0.1 -0.1)\">\r\n      <use xlink:href=\"#DejaVuSans-76\"/>\r\n      <use x=\"55.697266\" xlink:href=\"#DejaVuSans-101\"/>\r\n      <use x=\"117.220703\" xlink:href=\"#DejaVuSans-97\"/>\r\n      <use x=\"178.5\" xlink:href=\"#DejaVuSans-114\"/>\r\n      <use x=\"219.597656\" xlink:href=\"#DejaVuSans-110\"/>\r\n      <use x=\"282.976562\" xlink:href=\"#DejaVuSans-105\"/>\r\n      <use x=\"310.759766\" xlink:href=\"#DejaVuSans-110\"/>\r\n      <use x=\"374.138672\" xlink:href=\"#DejaVuSans-103\"/>\r\n      <use x=\"437.615234\" xlink:href=\"#DejaVuSans-32\"/>\r\n      <use x=\"469.402344\" xlink:href=\"#DejaVuSans-114\"/>\r\n      <use x=\"510.515625\" xlink:href=\"#DejaVuSans-97\"/>\r\n      <use x=\"571.794922\" xlink:href=\"#DejaVuSans-116\"/>\r\n      <use x=\"611.003906\" xlink:href=\"#DejaVuSans-101\"/>\r\n      <use x=\"672.527344\" xlink:href=\"#DejaVuSans-32\"/>\r\n      <use x=\"704.314453\" xlink:href=\"#DejaVuSans-100\"/>\r\n      <use x=\"767.791016\" xlink:href=\"#DejaVuSans-101\"/>\r\n      <use x=\"829.314453\" xlink:href=\"#DejaVuSans-99\"/>\r\n      <use x=\"884.294922\" xlink:href=\"#DejaVuSans-97\"/>\r\n      <use x=\"945.574219\" xlink:href=\"#DejaVuSans-121\"/>\r\n     </g>\r\n    </g>\r\n   </g>\r\n  </g>\r\n  <g id=\"axes_2\">\r\n   <g id=\"patch_8\">\r\n    <path d=\"M 469.890341 361.9155 \r\nL 824.98125 361.9155 \r\nL 824.98125 35.7555 \r\nL 469.890341 35.7555 \r\nz\r\n\" style=\"fill:#ffffff;\"/>\r\n   </g>\r\n   <g id=\"matplotlib.axis_3\">\r\n    <g id=\"xtick_8\">\r\n     <g id=\"line2d_20\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"486.030837\" xlink:href=\"#mb57ca4ec09\" y=\"361.9155\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_19\">\r\n      <!-- 0 -->\r\n      <g transform=\"translate(482.849587 376.513937)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_9\">\r\n     <g id=\"line2d_21\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"541.687719\" xlink:href=\"#mb57ca4ec09\" y=\"361.9155\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_20\">\r\n      <!-- 5 -->\r\n      <g transform=\"translate(538.506469 376.513937)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-53\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_10\">\r\n     <g id=\"line2d_22\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"597.344601\" xlink:href=\"#mb57ca4ec09\" y=\"361.9155\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_21\">\r\n      <!-- 10 -->\r\n      <g transform=\"translate(590.982101 376.513937)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-49\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_11\">\r\n     <g id=\"line2d_23\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"653.001484\" xlink:href=\"#mb57ca4ec09\" y=\"361.9155\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_22\">\r\n      <!-- 15 -->\r\n      <g transform=\"translate(646.638984 376.513937)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-49\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_12\">\r\n     <g id=\"line2d_24\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"708.658366\" xlink:href=\"#mb57ca4ec09\" y=\"361.9155\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_23\">\r\n      <!-- 20 -->\r\n      <g transform=\"translate(702.295866 376.513937)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-50\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_13\">\r\n     <g id=\"line2d_25\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"764.315248\" xlink:href=\"#mb57ca4ec09\" y=\"361.9155\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_24\">\r\n      <!-- 25 -->\r\n      <g transform=\"translate(757.952748 376.513937)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-50\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-53\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"xtick_14\">\r\n     <g id=\"line2d_26\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"819.972131\" xlink:href=\"#mb57ca4ec09\" y=\"361.9155\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_25\">\r\n      <!-- 30 -->\r\n      <g transform=\"translate(813.609631 376.513937)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-51\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"text_26\">\r\n     <!-- epoch -->\r\n     <g transform=\"translate(632.20767 390.192062)scale(0.1 -0.1)\">\r\n      <use xlink:href=\"#DejaVuSans-101\"/>\r\n      <use x=\"61.523438\" xlink:href=\"#DejaVuSans-112\"/>\r\n      <use x=\"125\" xlink:href=\"#DejaVuSans-111\"/>\r\n      <use x=\"186.181641\" xlink:href=\"#DejaVuSans-99\"/>\r\n      <use x=\"241.162109\" xlink:href=\"#DejaVuSans-104\"/>\r\n     </g>\r\n    </g>\r\n   </g>\r\n   <g id=\"matplotlib.axis_4\">\r\n    <g id=\"ytick_7\">\r\n     <g id=\"line2d_27\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"469.890341\" xlink:href=\"#m73aa8f3890\" y=\"328.697564\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_27\">\r\n      <!-- 0.86 -->\r\n      <g transform=\"translate(440.624716 332.496783)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-56\"/>\r\n       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-54\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_8\">\r\n     <g id=\"line2d_28\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"469.890341\" xlink:href=\"#m73aa8f3890\" y=\"272.962773\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_28\">\r\n      <!-- 0.87 -->\r\n      <defs>\r\n       <path d=\"M 8.203125 72.90625 \r\nL 55.078125 72.90625 \r\nL 55.078125 68.703125 \r\nL 28.609375 0 \r\nL 18.3125 0 \r\nL 43.21875 64.59375 \r\nL 8.203125 64.59375 \r\nz\r\n\" id=\"DejaVuSans-55\"/>\r\n      </defs>\r\n      <g transform=\"translate(440.624716 276.761991)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-56\"/>\r\n       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-55\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_9\">\r\n     <g id=\"line2d_29\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"469.890341\" xlink:href=\"#m73aa8f3890\" y=\"217.227981\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_29\">\r\n      <!-- 0.88 -->\r\n      <g transform=\"translate(440.624716 221.0272)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-56\"/>\r\n       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-56\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_10\">\r\n     <g id=\"line2d_30\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"469.890341\" xlink:href=\"#m73aa8f3890\" y=\"161.49319\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_30\">\r\n      <!-- 0.89 -->\r\n      <defs>\r\n       <path d=\"M 10.984375 1.515625 \r\nL 10.984375 10.5 \r\nQ 14.703125 8.734375 18.5 7.8125 \r\nQ 22.3125 6.890625 25.984375 6.890625 \r\nQ 35.75 6.890625 40.890625 13.453125 \r\nQ 46.046875 20.015625 46.78125 33.40625 \r\nQ 43.953125 29.203125 39.59375 26.953125 \r\nQ 35.25 24.703125 29.984375 24.703125 \r\nQ 19.046875 24.703125 12.671875 31.3125 \r\nQ 6.296875 37.9375 6.296875 49.421875 \r\nQ 6.296875 60.640625 12.9375 67.421875 \r\nQ 19.578125 74.21875 30.609375 74.21875 \r\nQ 43.265625 74.21875 49.921875 64.515625 \r\nQ 56.59375 54.828125 56.59375 36.375 \r\nQ 56.59375 19.140625 48.40625 8.859375 \r\nQ 40.234375 -1.421875 26.421875 -1.421875 \r\nQ 22.703125 -1.421875 18.890625 -0.6875 \r\nQ 15.09375 0.046875 10.984375 1.515625 \r\nz\r\nM 30.609375 32.421875 \r\nQ 37.25 32.421875 41.125 36.953125 \r\nQ 45.015625 41.5 45.015625 49.421875 \r\nQ 45.015625 57.28125 41.125 61.84375 \r\nQ 37.25 66.40625 30.609375 66.40625 \r\nQ 23.96875 66.40625 20.09375 61.84375 \r\nQ 16.21875 57.28125 16.21875 49.421875 \r\nQ 16.21875 41.5 20.09375 36.953125 \r\nQ 23.96875 32.421875 30.609375 32.421875 \r\nz\r\n\" id=\"DejaVuSans-57\"/>\r\n      </defs>\r\n      <g transform=\"translate(440.624716 165.292408)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-56\"/>\r\n       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-57\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_11\">\r\n     <g id=\"line2d_31\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"469.890341\" xlink:href=\"#m73aa8f3890\" y=\"105.758398\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_31\">\r\n      <!-- 0.90 -->\r\n      <g transform=\"translate(440.624716 109.557617)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-57\"/>\r\n       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-48\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"ytick_12\">\r\n     <g id=\"line2d_32\">\r\n      <g>\r\n       <use style=\"stroke:#000000;stroke-width:0.8;\" x=\"469.890341\" xlink:href=\"#m73aa8f3890\" y=\"50.023607\"/>\r\n      </g>\r\n     </g>\r\n     <g id=\"text_32\">\r\n      <!-- 0.91 -->\r\n      <g transform=\"translate(440.624716 53.822825)scale(0.1 -0.1)\">\r\n       <use xlink:href=\"#DejaVuSans-48\"/>\r\n       <use x=\"63.623047\" xlink:href=\"#DejaVuSans-46\"/>\r\n       <use x=\"95.410156\" xlink:href=\"#DejaVuSans-57\"/>\r\n       <use x=\"159.033203\" xlink:href=\"#DejaVuSans-49\"/>\r\n      </g>\r\n     </g>\r\n    </g>\r\n    <g id=\"text_33\">\r\n     <!-- Test acc -->\r\n     <defs>\r\n      <path d=\"M -0.296875 72.90625 \r\nL 61.375 72.90625 \r\nL 61.375 64.59375 \r\nL 35.5 64.59375 \r\nL 35.5 0 \r\nL 25.59375 0 \r\nL 25.59375 64.59375 \r\nL -0.296875 64.59375 \r\nz\r\n\" id=\"DejaVuSans-84\"/>\r\n     </defs>\r\n     <g transform=\"translate(434.545028 219.669094)rotate(-90)scale(0.1 -0.1)\">\r\n      <use xlink:href=\"#DejaVuSans-84\"/>\r\n      <use x=\"60.818359\" xlink:href=\"#DejaVuSans-101\"/>\r\n      <use x=\"122.341797\" xlink:href=\"#DejaVuSans-115\"/>\r\n      <use x=\"174.441406\" xlink:href=\"#DejaVuSans-116\"/>\r\n      <use x=\"213.650391\" xlink:href=\"#DejaVuSans-32\"/>\r\n      <use x=\"245.4375\" xlink:href=\"#DejaVuSans-97\"/>\r\n      <use x=\"306.716797\" xlink:href=\"#DejaVuSans-99\"/>\r\n      <use x=\"361.697266\" xlink:href=\"#DejaVuSans-99\"/>\r\n     </g>\r\n    </g>\r\n   </g>\r\n   <g id=\"line2d_33\">\r\n    <path clip-path=\"url(#p8e8bdd8bdf)\" d=\"M 486.030837 341.516566 \r\nL 497.162213 238.407202 \r\nL 508.29359 244.538029 \r\nL 519.424966 157.591754 \r\nL 530.556343 158.70645 \r\nL 541.687719 130.281706 \r\nL 552.819096 132.511098 \r\nL 563.950472 114.118617 \r\nL 575.081848 127.494967 \r\nL 586.213225 106.315746 \r\nL 597.344601 125.265575 \r\nL 608.475978 107.98779 \r\nL 619.607354 104.086354 \r\nL 630.738731 134.183142 \r\nL 641.870107 86.808569 \r\nL 653.001484 87.923265 \r\nL 664.13286 97.955527 \r\nL 675.264237 148.674188 \r\nL 686.395613 109.102486 \r\nL 697.52699 76.218959 \r\nL 708.658366 82.349786 \r\nL 719.789742 73.989567 \r\nL 730.921119 73.989567 \r\nL 742.052495 71.760175 \r\nL 753.183872 96.283484 \r\nL 764.315248 103.529006 \r\nL 775.446625 67.301392 \r\nL 786.578001 60.613217 \r\nL 797.709378 94.61144 \r\nL 808.840754 50.580955 \r\n\" style=\"fill:none;stroke:#1f77b4;stroke-linecap:square;stroke-width:1.5;\"/>\r\n   </g>\r\n   <g id=\"line2d_34\">\r\n    <path clip-path=\"url(#p8e8bdd8bdf)\" d=\"M 486.030837 347.090045 \r\nL 497.162213 325.910825 \r\nL 508.29359 235.063114 \r\nL 519.424966 169.29606 \r\nL 530.556343 136.969881 \r\nL 541.687719 158.70645 \r\nL 552.819096 109.659834 \r\nL 563.950472 105.758398 \r\nL 575.081848 97.398179 \r\nL 586.213225 81.792438 \r\nL 597.344601 87.923265 \r\nL 608.475978 81.23509 \r\nL 619.607354 70.088132 \r\nL 630.738731 75.104263 \r\nL 641.870107 61.727913 \r\nL 653.001484 66.744044 \r\nL 664.13286 63.399957 \r\nL 675.264237 60.613217 \r\nL 686.395613 61.170565 \r\nL 697.52699 60.613217 \r\nL 708.658366 62.285261 \r\nL 719.789742 59.498521 \r\nL 730.921119 62.842609 \r\nL 742.052495 63.399957 \r\nL 753.183872 61.727913 \r\nL 764.315248 60.613217 \r\nL 775.446625 59.498521 \r\nL 786.578001 60.055869 \r\nL 797.709378 59.498521 \r\nL 808.840754 58.941173 \r\n\" style=\"fill:none;stroke:#ff7f0e;stroke-linecap:square;stroke-width:1.5;\"/>\r\n   </g>\r\n   <g id=\"patch_9\">\r\n    <path d=\"M 469.890341 361.9155 \r\nL 469.890341 35.7555 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n   <g id=\"patch_10\">\r\n    <path d=\"M 824.98125 361.9155 \r\nL 824.98125 35.7555 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n   <g id=\"patch_11\">\r\n    <path d=\"M 469.890341 361.9155 \r\nL 824.98125 361.9155 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n   <g id=\"patch_12\">\r\n    <path d=\"M 469.890341 35.7555 \r\nL 824.98125 35.7555 \r\n\" style=\"fill:none;stroke:#000000;stroke-linecap:square;stroke-linejoin:miter;stroke-width:0.8;\"/>\r\n   </g>\r\n   <g id=\"text_34\">\r\n    <!-- Comparison of test acc by  -->\r\n    <g transform=\"translate(567.036733 16.318125)scale(0.12 -0.12)\">\r\n     <use xlink:href=\"#DejaVuSans-67\"/>\r\n     <use x=\"69.824219\" xlink:href=\"#DejaVuSans-111\"/>\r\n     <use x=\"131.005859\" xlink:href=\"#DejaVuSans-109\"/>\r\n     <use x=\"228.417969\" xlink:href=\"#DejaVuSans-112\"/>\r\n     <use x=\"291.894531\" xlink:href=\"#DejaVuSans-97\"/>\r\n     <use x=\"353.173828\" xlink:href=\"#DejaVuSans-114\"/>\r\n     <use x=\"394.287109\" xlink:href=\"#DejaVuSans-105\"/>\r\n     <use x=\"422.070312\" xlink:href=\"#DejaVuSans-115\"/>\r\n     <use x=\"474.169922\" xlink:href=\"#DejaVuSans-111\"/>\r\n     <use x=\"535.351562\" xlink:href=\"#DejaVuSans-110\"/>\r\n     <use x=\"598.730469\" xlink:href=\"#DejaVuSans-32\"/>\r\n     <use x=\"630.517578\" xlink:href=\"#DejaVuSans-111\"/>\r\n     <use x=\"691.699219\" xlink:href=\"#DejaVuSans-102\"/>\r\n     <use x=\"726.904297\" xlink:href=\"#DejaVuSans-32\"/>\r\n     <use x=\"758.691406\" xlink:href=\"#DejaVuSans-116\"/>\r\n     <use x=\"797.900391\" xlink:href=\"#DejaVuSans-101\"/>\r\n     <use x=\"859.423828\" xlink:href=\"#DejaVuSans-115\"/>\r\n     <use x=\"911.523438\" xlink:href=\"#DejaVuSans-116\"/>\r\n     <use x=\"950.732422\" xlink:href=\"#DejaVuSans-32\"/>\r\n     <use x=\"982.519531\" xlink:href=\"#DejaVuSans-97\"/>\r\n     <use x=\"1043.798828\" xlink:href=\"#DejaVuSans-99\"/>\r\n     <use x=\"1098.779297\" xlink:href=\"#DejaVuSans-99\"/>\r\n     <use x=\"1153.759766\" xlink:href=\"#DejaVuSans-32\"/>\r\n     <use x=\"1185.546875\" xlink:href=\"#DejaVuSans-98\"/>\r\n     <use x=\"1249.023438\" xlink:href=\"#DejaVuSans-121\"/>\r\n     <use x=\"1308.203125\" xlink:href=\"#DejaVuSans-32\"/>\r\n    </g>\r\n    <!-- LR decay and Batch size increasing -->\r\n    <g transform=\"translate(541.186108 29.7555)scale(0.12 -0.12)\">\r\n     <use xlink:href=\"#DejaVuSans-76\"/>\r\n     <use x=\"55.712891\" xlink:href=\"#DejaVuSans-82\"/>\r\n     <use x=\"125.195312\" xlink:href=\"#DejaVuSans-32\"/>\r\n     <use x=\"156.982422\" xlink:href=\"#DejaVuSans-100\"/>\r\n     <use x=\"220.458984\" xlink:href=\"#DejaVuSans-101\"/>\r\n     <use x=\"281.982422\" xlink:href=\"#DejaVuSans-99\"/>\r\n     <use x=\"336.962891\" xlink:href=\"#DejaVuSans-97\"/>\r\n     <use x=\"398.242188\" xlink:href=\"#DejaVuSans-121\"/>\r\n     <use x=\"457.421875\" xlink:href=\"#DejaVuSans-32\"/>\r\n     <use x=\"489.208984\" xlink:href=\"#DejaVuSans-97\"/>\r\n     <use x=\"550.488281\" xlink:href=\"#DejaVuSans-110\"/>\r\n     <use x=\"613.867188\" xlink:href=\"#DejaVuSans-100\"/>\r\n     <use x=\"677.34375\" xlink:href=\"#DejaVuSans-32\"/>\r\n     <use x=\"709.130859\" xlink:href=\"#DejaVuSans-66\"/>\r\n     <use x=\"777.734375\" xlink:href=\"#DejaVuSans-97\"/>\r\n     <use x=\"839.013672\" xlink:href=\"#DejaVuSans-116\"/>\r\n     <use x=\"878.222656\" xlink:href=\"#DejaVuSans-99\"/>\r\n     <use x=\"933.203125\" xlink:href=\"#DejaVuSans-104\"/>\r\n     <use x=\"996.582031\" xlink:href=\"#DejaVuSans-32\"/>\r\n     <use x=\"1028.369141\" xlink:href=\"#DejaVuSans-115\"/>\r\n     <use x=\"1080.46875\" xlink:href=\"#DejaVuSans-105\"/>\r\n     <use x=\"1108.251953\" xlink:href=\"#DejaVuSans-122\"/>\r\n     <use x=\"1160.742188\" xlink:href=\"#DejaVuSans-101\"/>\r\n     <use x=\"1222.265625\" xlink:href=\"#DejaVuSans-32\"/>\r\n     <use x=\"1254.052734\" xlink:href=\"#DejaVuSans-105\"/>\r\n     <use x=\"1281.835938\" xlink:href=\"#DejaVuSans-110\"/>\r\n     <use x=\"1345.214844\" xlink:href=\"#DejaVuSans-99\"/>\r\n     <use x=\"1400.195312\" xlink:href=\"#DejaVuSans-114\"/>\r\n     <use x=\"1441.277344\" xlink:href=\"#DejaVuSans-101\"/>\r\n     <use x=\"1502.800781\" xlink:href=\"#DejaVuSans-97\"/>\r\n     <use x=\"1564.080078\" xlink:href=\"#DejaVuSans-115\"/>\r\n     <use x=\"1616.179688\" xlink:href=\"#DejaVuSans-105\"/>\r\n     <use x=\"1643.962891\" xlink:href=\"#DejaVuSans-110\"/>\r\n     <use x=\"1707.341797\" xlink:href=\"#DejaVuSans-103\"/>\r\n    </g>\r\n   </g>\r\n   <g id=\"legend_2\">\r\n    <g id=\"patch_13\">\r\n     <path d=\"M 476.890341 73.11175 \r\nL 609.368466 73.11175 \r\nQ 611.368466 73.11175 611.368466 71.11175 \r\nL 611.368466 42.7555 \r\nQ 611.368466 40.7555 609.368466 40.7555 \r\nL 476.890341 40.7555 \r\nQ 474.890341 40.7555 474.890341 42.7555 \r\nL 474.890341 71.11175 \r\nQ 474.890341 73.11175 476.890341 73.11175 \r\nz\r\n\" style=\"fill:#ffffff;opacity:0.8;stroke:#cccccc;stroke-linejoin:miter;\"/>\r\n    </g>\r\n    <g id=\"line2d_35\">\r\n     <path d=\"M 478.890341 48.853937 \r\nL 498.890341 48.853937 \r\n\" style=\"fill:none;stroke:#1f77b4;stroke-linecap:square;stroke-width:1.5;\"/>\r\n    </g>\r\n    <g id=\"line2d_36\"/>\r\n    <g id=\"text_35\">\r\n     <!-- Batch size increase -->\r\n     <g transform=\"translate(506.890341 52.353937)scale(0.1 -0.1)\">\r\n      <use xlink:href=\"#DejaVuSans-66\"/>\r\n      <use x=\"68.603516\" xlink:href=\"#DejaVuSans-97\"/>\r\n      <use x=\"129.882812\" xlink:href=\"#DejaVuSans-116\"/>\r\n      <use x=\"169.091797\" xlink:href=\"#DejaVuSans-99\"/>\r\n      <use x=\"224.072266\" xlink:href=\"#DejaVuSans-104\"/>\r\n      <use x=\"287.451172\" xlink:href=\"#DejaVuSans-32\"/>\r\n      <use x=\"319.238281\" xlink:href=\"#DejaVuSans-115\"/>\r\n      <use x=\"371.337891\" xlink:href=\"#DejaVuSans-105\"/>\r\n      <use x=\"399.121094\" xlink:href=\"#DejaVuSans-122\"/>\r\n      <use x=\"451.611328\" xlink:href=\"#DejaVuSans-101\"/>\r\n      <use x=\"513.134766\" xlink:href=\"#DejaVuSans-32\"/>\r\n      <use x=\"544.921875\" xlink:href=\"#DejaVuSans-105\"/>\r\n      <use x=\"572.705078\" xlink:href=\"#DejaVuSans-110\"/>\r\n      <use x=\"636.083984\" xlink:href=\"#DejaVuSans-99\"/>\r\n      <use x=\"691.064453\" xlink:href=\"#DejaVuSans-114\"/>\r\n      <use x=\"732.146484\" xlink:href=\"#DejaVuSans-101\"/>\r\n      <use x=\"793.669922\" xlink:href=\"#DejaVuSans-97\"/>\r\n      <use x=\"854.949219\" xlink:href=\"#DejaVuSans-115\"/>\r\n      <use x=\"907.048828\" xlink:href=\"#DejaVuSans-101\"/>\r\n     </g>\r\n    </g>\r\n    <g id=\"line2d_37\">\r\n     <path d=\"M 478.890341 63.532062 \r\nL 498.890341 63.532062 \r\n\" style=\"fill:none;stroke:#ff7f0e;stroke-linecap:square;stroke-width:1.5;\"/>\r\n    </g>\r\n    <g id=\"line2d_38\"/>\r\n    <g id=\"text_36\">\r\n     <!-- Learning rate decay -->\r\n     <g transform=\"translate(506.890341 67.032062)scale(0.1 -0.1)\">\r\n      <use xlink:href=\"#DejaVuSans-76\"/>\r\n      <use x=\"55.697266\" xlink:href=\"#DejaVuSans-101\"/>\r\n      <use x=\"117.220703\" xlink:href=\"#DejaVuSans-97\"/>\r\n      <use x=\"178.5\" xlink:href=\"#DejaVuSans-114\"/>\r\n      <use x=\"219.597656\" xlink:href=\"#DejaVuSans-110\"/>\r\n      <use x=\"282.976562\" xlink:href=\"#DejaVuSans-105\"/>\r\n      <use x=\"310.759766\" xlink:href=\"#DejaVuSans-110\"/>\r\n      <use x=\"374.138672\" xlink:href=\"#DejaVuSans-103\"/>\r\n      <use x=\"437.615234\" xlink:href=\"#DejaVuSans-32\"/>\r\n      <use x=\"469.402344\" xlink:href=\"#DejaVuSans-114\"/>\r\n      <use x=\"510.515625\" xlink:href=\"#DejaVuSans-97\"/>\r\n      <use x=\"571.794922\" xlink:href=\"#DejaVuSans-116\"/>\r\n      <use x=\"611.003906\" xlink:href=\"#DejaVuSans-101\"/>\r\n      <use x=\"672.527344\" xlink:href=\"#DejaVuSans-32\"/>\r\n      <use x=\"704.314453\" xlink:href=\"#DejaVuSans-100\"/>\r\n      <use x=\"767.791016\" xlink:href=\"#DejaVuSans-101\"/>\r\n      <use x=\"829.314453\" xlink:href=\"#DejaVuSans-99\"/>\r\n      <use x=\"884.294922\" xlink:href=\"#DejaVuSans-97\"/>\r\n      <use x=\"945.574219\" xlink:href=\"#DejaVuSans-121\"/>\r\n     </g>\r\n    </g>\r\n   </g>\r\n  </g>\r\n </g>\r\n <defs>\r\n  <clipPath id=\"p4f5a07b44d\">\r\n   <rect height=\"326.16\" width=\"355.090909\" x=\"43.78125\" y=\"35.7555\"/>\r\n  </clipPath>\r\n  <clipPath id=\"p8e8bdd8bdf\">\r\n   <rect height=\"326.16\" width=\"355.090909\" x=\"469.890341\" y=\"35.7555\"/>\r\n  </clipPath>\r\n </defs>\r\n</svg>\r\n",
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA50AAAG8CAYAAABdQ8HwAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAAMTQAADE0B0s6tTgAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3xN9//A8dfNzbiZZCAhYkUShNirKDWrNkW1VlGtqrZmi5aqtqjSqvrKzx6p2doURZTaI2YlkWE1Vqzs3CSf3x9Xbl2SCLLwfj4e98E953M+53POvSef+z6fcTRKKYUQQgghhBBCCJELzPK7AEIIIYQQQgghXlwSdAohhBBCCCGEyDUSdAohhBBCCCGEyDUSdAohhBBCCCGEyDUSdAohhBBCCCGEyDUSdAohhBBCCCGEyDUSdAohhBBCCCGEyDUSdAqRxzQaDX/++Wd+FyNT3377LUWLFkWj0RAYGJjfxclUQT+PQgghckZB/3v/vNSbuWn8+PE0aNAgv4shCjAJOkWBFxkZSZ8+fShevDg6nQ4vLy+GDBnC5cuX87toTyUqKopGjRrldzEydOHCBcaOHcv//d//ERUVRf369R9JIxWLEEIUbFJv5p3s1Jtjx46lcePGObbP8+fPo9FoiIyMzLE8hchtEnSKAi04OJiaNWsSHR3NihUrCAkJYdGiRaSkpDB9+vT8Lt4TSUpKAsDV1RVLS8t8Lk3GIiIiUErRvn37Al1OIYQQGZN6M29JvSlE9kjQKQq0Dz/8kHLlyrF+/XoaNmyIh4cH9erVY9asWXzxxRfGdN9//z0lS5bEysqKunXrcujQIeO6hQsX4u7uzvLlyylTpgx2dnZ89NFHpKam8sUXX+Ds7Iy7uztLly41bhMYGIhGo2Hz5s14eXlhbW1Np06duHPnjjHNvHnzqFq1Kra2tpQqVYovvviClJQU4/o+ffrw9ttv89lnn+Hi4kKXLl0A025C0dHRvPnmmzg5OWFra4ufnx/79+835rF48WI8PT2xsrKicuXKbNmy5ZEy7tixg4oVK2Jvb0+HDh24fft2luc0s3O1cOFCmjRpAoCZmRkajSb7H9QDDh06RL169bCysqJkyZJMmTLFuE4pxeeff06JEiXQ6XSULVsWf39/ABITExkwYABFixbF2toaHx8f1q5dm+W+IiIiaNiwITqdjpo1a3Lq1CkADhw4gE6ne+RcNGzYkPHjxz/VcQkhxPNA6s2CVW8uXLiQb775ht27d6PRaExaKI8dO0bjxo2xtramdOnSjBs3zng+sqovy5cvD0CZMmXQaDSZ1mvr16+nbt262NvbU7x4cQYNGkRcXJxJmuXLl1O5cmWsrKxwd3dn4sSJxnVhYWG0b98eBwcHChUqRLNmzbJ1rooWLYqjoyOjR49GKQVAq1atGDlypEnaHTt2YGdnR2xsbJZ5iheEEqKAunHjhtJoNGr58uVZpgsICFA2NjZq6dKl6uzZs2rAgAHK2dlZ3b17Vyml1IIFC5ROp1Nt27ZVp06dUhs3blSWlpaqefPmavTo0So4OFhNnDhR6XQ6df36daWUUrt27VKAqlmzptq3b5/av3+/qlChgurdu7dxv//3f/+ntm/frsLDw9XmzZuVq6ur+uWXX4zre/furWxtbdWQIUPUuXPnVEhIiFJKKUBt375dKaXUBx98oFq2bKlOnTqlzp8/r3777Td15MgRpZRSf//9t9Jqteqnn35S586dU1988YWytLRUERERJmVs3LixOnjwoDp8+LAqW7asGjp06FOdq/j4eLVy5UoFqKioKBUVFZVhHuPGjVOvvPJKhuvu3bunnJ2dVb9+/dTZs2fVr7/+qmxsbFRAQIBSSqkVK1YoDw8PtXfvXhUZGal27typ1qxZo5RSavLkyapatWrqyJEjxnO6Y8eOTI8FUK6urmrlypXqzJkzqkuXLqpcuXIqJSVFKaWUj4+Pmj17tjF9eHi40mg06vz585nmKYQQzzOpNwtevRkfH68++eQTVa9ePWOalJQUdfPmTeXk5KQmT56sQkND1a5du5Snp6eaNGmSUirr+nL//v0KUIcOHVJRUVEqJiYmw7KvWLFCrV+/XoWFhandu3crHx8fNWLECOP6rVu3KgsLC/XDDz+okJAQtW/fPjV37lyllFKJiYmqbNmyqk2bNurIkSPqn3/+Ub/88ou6ceNGhvsaN26csrOzUx07dlSnT59Wq1atUvb29mrBggVKKaWWLVumihcvrlJTU00+7549e2Z67sWLRYJOUWAdOHBAAer48eNZpqtTp47JH1G9Xq/c3d3VzJkzlVKGylOj0airV68a07Rs2VJVqlTJ+D4lJUXZ2tqq9evXK6X+q5i2bNliTLN9+3Zlbm6ubt++nWE5vvvuO9WkSRPj+969e6uyZcua/IFVyrTybNOmjZowYUKG+XXr1k29+eabjxzr8OHDTcp48OBB4/pvv/1W1ahRI8P80rfP6lxt375dPe5eVFZB5//+9z9VvHhxpdfrjctGjRqlatasqZRSaurUqapp06YqLS3tkW0HDx6s3n333Sz3/SBAjRo1yvj+zp07ysbGRm3YsEEpZfg86tevb1z/1VdfZVpuIYR4EUi9WTDrzTFjxqhXX33VZNlXX32lOnfubLIsICBAlStXTimVdX0ZGhqqAGMwnV3Lli1TZcqUMb5v1KiR+vDDDzNMO3/+fFWkSBEVFxeXrbzHjRunrK2t1a1bt4zLxowZYzy3CQkJqnDhwmrbtm1KKaXi4uKUnZ2d8XMVLz7pXiuee8HBwdStW9f43tzcnJo1axIcHGxcVqRIEYoVK2Z8X6xYMSpVqmR8r9VqcXZ25saNGyZ5165d2+T/KSkphIWFAbBv3z5atGhBiRIlsLOzY/z48Vy6dMlkez8/P8zMMr/MBgwYwLfffkvDhg2ZMGGCSZkfPi6AevXqmaQBqFy5svH/rq6uXL9+PdP9ZedcPYvg4GBq1KiBubl5hmXu3LkzZ8+epUKFCnz66afs3r3bmK5nz56sXr2aGjVqMHr0aI4ePfrY/T34+RQqVAhvb2/jvnr16sXBgweNn9fSpUvp1atXjhynEEI8z6TezP9689SpU6xfvx47Ozvjq1+/fkRGRpKWlpZlfZldZ8+epWPHjnh4eGBvb0/fvn1Nzvfp06czneDo9OnT1K5dGxsbm2zvz9PTE0dHR+P72rVrG8+TTqejW7duLFmyBIA1a9ZQuHBhXnvttSc+LvF8kqBTFFjlypVDo9HkSEBkYWFh8l6j0WS4LC0t7ZFlGf0/JiaGN954gzJlyvDbb79x7NgxPvvsM/R6vcn2j/tj3a5dO8LDw+nZsyfHjh2jSpUqrFixAsA4DuJJji2jY8hLjytz6dKlCQ0NZeLEicTGxtK2bVs++ugjwFA5RURE8Mknn3DhwgVeeeUVpk6dmmV+WY07LV68OM2aNWPJkiXs37+fS5cu0bVr1yc/KCGEeE5Ivfn81JuxsbF0796doKAg4+vUqVOcO3cOMzOzLOvL7GrXrh0ajYaAgACOHDnCjBkzTMbQZiW75/JBj5sLok+fPvz+++/ExcWxePFi3nnnnSxvMIgXi3zSosBycXGhSZMm/Pjjjxn+8bt79y4A3t7eHDhwwLg8JSWFI0eO4OPj88xleHBihUOHDmFubk65cuUIDg7mzp07TJ48mbp16+Ll5fXI3drscnNz47333mPt2rX069ePRYsWAeDj42NyXAD79+9/puPKzXMFhjIfPXrUpFJ7uMy2trZ06dKFOXPmMHfuXObNm2dc5+TkRM+ePQkICGDChAnMnz8/y/09+Pncu3ePkJAQvL29jcv69u3L0qVLWbx4Me3ataNw4cI5cZhCCFEgSb1ZMOtNCwsLUlNTTZb5+flx9uxZPD09H3mly6y+TA+aH87zQTdv3iQsLIwvv/yShg0b4u3tzdWrV03S+Pr6Zvpc0cqVK3P48GHi4+OzfZyhoaEmE0cdPnzYpE6uW7cuHh4e/PLLL+zYsUN6H71kzB+fRIj8M3PmTF555RWaNWvGqFGj8PLy4tq1ayxduhRLS0t++OEHPv74YwYMGEDVqlWpXr0606ZNIyEhgXfeeeeZ9//FF18YA5WPP/6YHj16ULhwYTw8PLCwsGDWrFl0796dbdu2sXbtWuzt7Z8o/3HjxlGrVi0qVqzIrVu3+Pvvv41dTYYMGUKjRo2YOXMmLVq0YOnSpRw/fpzly5c/9fHk1LmKi4sjKCjIZJmrqytvv/02Y8eO5YMPPmDYsGEcP36cn3/+mTlz5gCwaNEilFLUqVMHrVbL2rVrjRXS9OnTcXd3p2rVqiQmJrJt2zaTyiojixYtokaNGlSqVInx48dTrFgxWrVqZVzfoUMH3n//febNm8eaNWue6BiFEOJ5JPVmwas3S5UqRXBwMOfOncPFxQUnJyc+/PBD/P39GTBgAIMHD0an03HixAlCQkIYO3ZslvVl+qNZtm3bRpcuXbC1tX2khdjR0RFHR0fmzJnDsGHDOHz4sHH223RjxoyhTZs2lCtXjjZt2nD79m3OnDlD37596dGjBxMnTqRbt26MHz8ee3t7du3aRefOnXFxccnwOLVaLf3792fChAmcO3eOGTNm8OOPP5qk6d27N2PHjqVatWpUqFAh2+dQvADybzipENkTFhamevXqpVxdXZWVlZXy9PRUH330kbp8+bIxzZQpU1SJEiWUpaWlqlOnjskkAQsWLFAlSpQwybN3797q7bffNllWqlQpNWfOHKXUf5MNrF+/XpUrV05ZWVmp9u3bmwyQT8/XxsZGdezYUU2ZMkWVKlUqy30oZTohwoQJE5S3t7eysrJSRYsWVf3791exsbHGtIsWLVLlypVTFhYWytfXV23evNm4Lr2MD07ak9GxPiyrc5XdiYSAR17jxo1TSil18OBBVadOHWVpaalKlCihJk+ebNx2zZo1qlatWsrOzk4VKlRItWzZUp07d04ppZS/v7+qXLmysra2Vk5OTurNN9/MdAbd9PPo7++v6tWrpywtLVW1atVUUFDQI+kGDhyoihYtanKehBDiRSb1ZsGqN2NiYtQbb7yh7OzsTCYAOnnypGrZsqWytbVV9vb2qlatWmrRokVKqazrS6WU+umnn5Srq6vSaDTG+vdhmzdvVp6enkqn06kmTZqoefPmPVLWpUuXKh8fH2VhYaHc3d3Vt99+a1wXGhqqWrdurWxsbJSDg4Nq0aJFppNCpU8y+N133ylnZ2dVqFAhNWrUqEcmQrpy5YrSaDRqxowZWZ4z8eLRKPUUnbaFeMEFBgbSpEkT9Hq9yaQ44vnTrl07PD09mTZtWn4XRQghXlhSb4rsOHbsGPXq1ePKlSuZtpiKF5P8VRBCvJDu3r3LX3/9xR9//MGpU6fyuzhCCCHES0uv13PlyhXGjRuXZRdd8eKSoFMI8UJq3749R44cYfz48Y8dGyqEEEKI3JM+9tbX15eNGzfmd3FEPpDutUIIIYQQQgghco08MkUIIYQQQgghRK6RoFPkuj///POxDwx+0RTUY164cCHu7u45ll/jxo0ZO3ZsjuX3oJwua34oXbo0c+fOze9iCCFeYAW1vslNBfWYpY7NW1LHPl8k6HzJZfUHbeHChWg0GjQaDVqtFg8PDz7++OMnelCwyFhgYKDx3Go0Guzt7alfvz67du3Kdh4pKSloNJpMH+ycF37//Xc+++yzXMm7W7duHD9+PFfyziuHDx/m7bffzu9iCCHyidSx+UPq2MeTOlbkNZlISGTJzc2NY8eOkZaWxj///EPfvn2xsLBg6tSp+V20F8Lly5fRarXExMQwc+ZM2rVrx6VLl4wP1i7onJycci1va2trrK2tcy3/pKQkrKysci1/gCJFiuRq/kKI55vUsblL6tjMSR0r8pq0dIosmZmZ4erqSvHixWnatClvvvkmO3bsyHKbU6dOUbNmTXQ6HQ0aNCAyMvKRNMuXL6dixYpYW1vj6+vL6tWrTdYfPXqU1157DRsbG5ycnOjQoYNx3XfffUeFChWwsbGhfPnyzJgxw7hu0qRJ1K5d2ySvmJgYbG1tM71buX79eurWrYu9vT3Fixdn0KBBxMXFGdePHz+eBg0aMHPmTNzc3HBxcWHkyJE8OAdXdo45I8WKFcPV1ZXy5cvz1VdfERsbS2hoKGD4g92rVy9KliyJra0tNWrUYOfOncZtPT09AWjSpAkajYY+ffoAhruzX375JR4eHuh0OipUqMCGDRtM9rt69WrKlClD4cKFeffdd0lKSsq0jNu3b6datWpYW1vj4uLCG2+8YVz34F388ePHm9xZTn+ln/f4+Hg+/PBDihQpQuHChWnTpk2W5+nhrj99+vThnXfeYezYsTg5OVG8ePFHnr0ZFhZG+/btcXBwoFChQjRr1ozbt28byzp8+HAGDBiAg4MDw4YNAyA8PJy2bdtiZ2dH8eLFGTx4sElLQ1bfN4Bly5bh4+ODTqfD1dWV9957z7juwa4/kZGRaDQa1q5dS+3atbG1taVx48ZcvHjRmD4xMZG+fftia2tLyZIlWbJkCe7u7ixcuDDT8ySEeH5JHSt1rNSxUse+NJR4qb366qtqzJgxGa5bsGCBKlGihPH9hQsXVKVKlVTt2rUzzS8lJUV5enqqjh07qtOnT6uVK1cqV1dX9eBXbceOHcrFxUWtXLlShYWFqYCAAGVtba3279+vlFLq+vXrqlChQqpfv37q5MmT6uTJk2rSpEnG7X/44Qe1Z88eFR4erlasWKFsbW3Vpk2blFJKXb58WWm1WnXu3DmT4yhVqpRKS0vLsMwrVqxQ69evV2FhYWr37t3Kx8dHjRgxwrh+3Lhxyt7eXvXq1UudPXtWrVmzRllYWKj169dn+5gftmvXLgUovV6vlFJKr9ern376STk4OKg7d+4opZSKjY1VX3/9tTp+/LgKDQ1V48ePV3Z2duratWtKKaWioqIUoH777TcVFRVl3G706NHK1dVV/fbbb+r8+fNq8+bNasuWLcZzodPpVJs2bdTJkyfVzp07lZOTk5oxY0aG5dTr9crBwUH9+OOPKjIyUp04cUJNnz7duP7B709MTIyKiooyvkaPHq2KFCmi/v33X6WUUj179lTNmzdXhw8fVufOnVN9+/ZVvr6+KiUlJcN9P/z96927t7K3t1cjR45UwcHByt/fXwHqxIkTSimlEhMTVdmyZVWbNm3UkSNH1D///KN++eUXdePGDWNZ7ezs1HfffadCQ0NVeHi4SkpKUp6enurTTz9V586dU4cOHVK1a9dW77//vnG/WX3f/v33X2VlZaWWL1+uIiMj1eHDh9Xs2bON25YqVUrNmTNHKaVURESEApSfn5/auXOnOn36tKpVq5bq1KmTMf2YMWNUiRIl1Pbt21VQUJBq0qSJ0ul0asGCBZl8k4QQBZnUsVLHSh0rdawwkKDzJfe4ClGj0ShbW1tlbW2tAKXRaNTy5cszzW/z5s3K2tpa3bp1y7hs1KhRJpVDkyZN1M8//2yy3YABA1S/fv2UUkp9+eWXytfXN9MK7GEDBw5Uffv2Nb5v2bKlGj16tMn+MjvGjCxbtkyVKVPG+H7cuHHK0dFRJSQkGJe1aNFCDRs2TCmVvWN+WHqFaGtrq2xtbZWZmZnJH9rMeHt7q0WLFimlDJUVoHbt2mVcHx8fr6ysrNSqVasy3D79M7169apx2Xvvvac6d+6cYfqbN28qQF28eDHD9Zl9fwIDA5VOpzOWLSIiQllaWpqco+TkZGVjY6P27NmTaVkfrhArVqxoksbLy8v4XZo/f74qUqSIiouLy7SsjRs3Nlm2aNEiVaNGDZNlf//9t7K0tMy0on7w+3bkyBHl4OCgYmJiMkybUYW4YsUK4/pff/1VOTs7G9+7uLgY0yulVHBwsAKkQhTiOSV17KOkjv2P1LGPkjr2xSXda0WWihUrRlBQEIcOHeLTTz/lrbfeolu3bpmmDw4OxtPTE0dHR+Oyh7vinDp1ihEjRmBnZ2d8LVy4kPDwcABOnz7Nq6++munMdJs2baJBgwYUK1YMOzs75s+fz6VLl4zr+/Tpw9KlS1FKcenSJXbv3k2vXr0yLfPZs2fp2LEjHh4e2Nvb07dvX5P8AMqXL49OpzO+d3V15fr169k+5swcPXqUoKAgjh07xtChQ+nRo4ex6w/A1KlTqVKlCk5OTtjZ2REaGvpI2R50/vx5kpKSaNy4caZpihQpQrFixTI8loc5OzvTvXt3fH196d69OwsWLCA2NjbLY7p8+TJdu3Zl8uTJxnKcOXMGvV5PyZIljZ+5o6MjCQkJxs89O3x9fU3eP1j206dPU7t2bWxsbDLdvlq1aibvT506xYkTJ0y+i82bNyc5OZkrV64AWX/f/Pz8qFKlCmXLlqVPnz6sXLmS5OTkLI+hcuXKJuWPjo4mNTWVO3fucPPmTWrUqGFc7+Xlhb29fTbOjBDieSR1rIHUsVLHSh374pOgU2RJq9Xi6emJr68v06ZN4+LFi/j7+2eaXin12GnMY2NjmTp1KkFBQcbX2bNnWbJkiTGPzISHh9OpUydee+01Nm3axPHjx+nVqxd6vd6YpkOHDty7d4/du3ezdOlSateujZeXV6Z5tmvXDo1GQ0BAAEeOHGHGjBmkpKSYpLGwsDB5r9FoSEtLy/YxZ6ZcuXJ4enri5+fHhAkTcHNzY8GCBQAsXbqUCRMmMGzYMHbt2kVQUBAVK1Y0OdaHZXXusnMsGVm2bBnbtm3D29ubqVOn4uvrS3R0dIZpk5KS6NSpE61atWLIkCHG5bGxsVhbW5t85kFBQYSEhJiMJXqWsmfn2B+uLGNjY2nUqJFJmU6cOEFoaChubm6P/b6Zm5sTGBjIihUrKFasGCNHjqR+/fpZVooPHkP690YZep2YLBNCvPikjjWQOlbqWKljX3wye614Ip999hnvvvsuPXv2zPBul7e3N6Ghody5c8c4O9zhw4dN0vj5+REeHm4cpP+wypUrs2bNmgwrmmPHjmFtbc2ECROMyyIiIkzS6HQ6unfvzpIlS9i/fz8fffRRpsdz8+ZNwsLCWL16NVWrVgVg5cqVWZyBR2XnmLPL3NycxMREAA4cOMBrr71G7969AcMf7wcHxGu1WszMzEhNTTUuK1++PFZWVgQGBtKlS5enKkNG6tSpQ506dRgzZgxFixZlx44ddO3a9ZF0H3zwAUqpR340+fn5ER8fT0JCgsldyJxUuXJlAgICiI+Pz/JO7MPlWr9+Pe7u7iZ32dNl5/um1Wpp0qQJTZo0YdiwYcaWi+zeiU/n6OiIi4sLR48eNX4XQ0NDiYmJeaJ8hBDPL6ljHyV17H+kjpU69nkmLZ2Ca9euPXJ3LLPuHa+//jr29vaZ3olt2bIlbm5u9O/fn7Nnz7J69WoWLVpkkmb06NH88ssvTJ8+nZCQEE6cOMHMmTNZsWIFAIMHD+bixYsMGDCAU6dOcfbsWeP08eXKlePevXssXLiQ8+fPM3HixAwrnz59+rBkyRLCwsKy7Krk6OiIo6Mjc+bMITw8nBUrVmR5l/lpjzkz165d4+rVq4SFhTFlyhTOnDlD69atjce6b98+9uzZw5kzZ+jdu7fJ3VKNRkPJkiXZuXMn169fN97pHDZsGEOGDOH3338nIiKCbdu28ccffzzRMaWLiIhgzJgxHDx4kAsXLrBq1SpiY2MpX778I2nnzp3L6tWrmTVrFnfu3OHq1atcvXqV5ORkfHx86NSpE927d2fr1q1ERETw119/8dFHH2V6R/dJ9ejRAzs7O7p168bRo0cJCQnB39+fmzdvZrrN22+/jaWlJd26dePw4cOcP3+eDRs2MHz4cODx37eDBw8yefJkjh07xoULF1i8eDFWVlaUKlXqqY5h4MCBjB8/np07d3Ly5EkGDRqETqeTO7NCPMekjpU6NjNSx0od+zKRoFMwd+5cqlWrZvI6cuRIhmnNzMwYNGgQU6dOzXAKcK1Wa/xDXK1aNaZNm8b48eNN0rRr145ly5axZMkSKleuTLNmzdi4caPxj0iRIkX4888/CQkJoVatWjRs2JB9+/YBhvEC33zzDSNHjqR69epERkYycODAR8pRp04dypYtS5s2bbJ8zpVWqyUgIIBt27ZRqVIl/P39Te64ZUd2jjkz7u7uuLm5Ge8gLl68mGbNmgHw/vvv07RpU1q3bk3z5s1p2LAhfn5+JttPmTKFgIAA3NzcGDx4MABfffUVffv2ZfDgwVSoUIFPP/3U5E7tk7CxseH06dO0b98eb29vvvnmG+bPn//IuA2AvXv3EhMTQ+3atXFzczO+0j+7gIAAWrVqxbvvvouPjw99+vRBr9dn+47p41hZWbF161bS0tJo1KgRtWrV4vfff8fcPPMOHfb29gQGBmJpaUnz5s3x8/Nj7NixuLm5AY//vjk4OLBjxw5atGhBhQoVWLZsGb///rvJeJ4nMXbsWJo2bUrbtm1p3bo1vXv3xsbGJtefdSaEyD1Sx0odmxmpY6WOfZloVHY6aQvxnElOTqZEiRLMmzePdu3a5XdxhHgqly5dwsPDg0OHDlGrVq38Lo4QQgBSx4oXg9SxeUvGdIoXzrVr15g1axbW1tYmD1kWoqALCwtj7969vPLKK9y6dYuRI0fi4+NDzZo187toQggBSB0rnl9Sx+YvCTrFC8fV1dU4Q51Wq83v4giRbRqNhlmzZvHhhx+i0+lo0KABS5YskfEmQogCQ+pY8bySOjZ/SfdaIYQQQgghhBC5RiYSEkIIIYQQQgiRayToFEIIIYQQQgiRa577MZ1WVlYUKVIkv4shhBCigLpx40aGj58Qjyd1rBBCiMfJTj373AedRYoU4fLly/ldDCGEEAWUu7t7fhfhuSV1rBBCiMfJTj0r3WuFEEIIIYQQQuQaCTqFEEIIIYQQQuQaCTqFEEIIIYQQQuSa535MpxDi+ZSWloY8JljkFI1Gg5mZ3EfNa3Idi5yi0WiMLyHEi0eCTiFEnkpOTubixYvo9fr8Lop4wVhYWODh4YGlpWV+F+WFJ9exyA0ajYbChQtTtGhRuYkkxAtGgk4hRJ66ePEi9vb2ODs7yx1tkWOUUkRHR3Px4kU8PT3zuzgvPLmORdU2UJYAACAASURBVG7Q6/Vcu3aNCxcuUKZMmfwujhAiB0nQKYTIM2lpaej1epydnTE3lz8/Imc5Oztz69Yt0tLSpJUkF8l1LHKLVqulRIkShIaGynUsxAsm16/mIUOGULp0aTQaDadPn84y7Y0bNyhWrBhdunTJ7WIJIfJB+tgvaRkRuSH9eyVjDHOXXMciN8l1LMSLKdeDzi5durB3715KlSr12LSDBg2idevWuV0kIYQQQgghhBB5JNeDzkaNGuHu7v7YdAEBARQrVoxXX301t4skhBAmSpcujY+PD1WrVqVChQr06NGDuLi4x263du1aDh069Nh0kZGRuLi4PFMZ+/fvz549e54pj3Tr169nxIgROZKXEAWBXMNCCFGwFYjO8v/++y/Tpk1j0qRJ+V0UIcRLavXq1QQFBXH27Fnu3bvHwoULH7tNdn+w5oS5c+fSsGHDHMmrXbt2fP/99zmSF0BKSkqO5SXE05Jr+OnJNSyEyG0FYgaAAQMGMGXKFOzs7B6bdtq0aUybNs34PjY2NjeLJoTIZf0XHeZCdHyu5F3K2Ya5vWs90TZJSUnExcXh6OgIwKlTpxg0aBBxcXEkJibSs2dPPv/8czZv3sz69ev5888/mTt3LoMHD6Z///4sWLCAn376CaUUFhYWrF692pj3l19+yaZNm7h79y4zZszIcDjBhg0bGDNmDGZmZqSkpPDNN9/Qvn17GjduzPDhw2nTpg01a9Y0/ki8fv063t7e7Nq1i6tXrzJkyBAiIyNJTEykQ4cOTJgw4ZF9LFy4kI0bN7J69WoCAwP55JNPqF+/Pn///TcpKSksWrSImjVrArBp0ybGjx9PcnIyGo0Gf39/6tSpg0ajYerUqWzYsIFatWrx/fffM3XqVFauXElKSgqurq74+/tTsmRJduzYwdixY0lMTESv1zNs2DD69u0LwMSJEwkICMDKygqAdevWUapUKQ4fPsyoUaO4d+8eaWlpjBkzhs6dOz/RZ/kiCQ0NpXfv3ty8eZPChQuzcOFCKlasaJImLi6OwYMHc+TIEZKTk+nYsSPfffcdGo2GyMhI+vTpw/HjxylfvjxHjhzJsbLl5jUMT34dyzUs17AQouApEEHn/v376devH2AIIhMSEmjZsiVbt259JO3QoUMZOnSo8X12uu4KIcTjdOnSBZ1OR0REBDVq1KBr166Aodven3/+iZWVFQkJCdSvX5/mzZvTunVr2rVrR82aNRk8eDAAgYGBfPPNN+zZswc3Nzfi4w0/xK9fv050dDQ1atRgwoQJ/PHHH3z88ccZ/mAdO3Yss2fPpn79+qSlpXHv3r1H0qQHDFFRUTRq1IixY8cC0Lt3b8aMGUOjRo1ISUmhTZs2rFmzho4dO2Z57GfOnGHu3LnMmjWL2bNnM2bMGLZu3UpISAj9+vXjr7/+wsvLC71ebzwmMPy4DwwMBODXX38lJCSE/fv3o9VqWbJkCYMHD2bdunVUr16dvXv3otVquXXrFtWrV6dVq1bodDqmTp1KVFQU1tbWxMfHY2Zmxp07dxg4cCCbNm3Czc2NmzdvUqNGDV555RVcXV2f8JN9MQwcOJD33nuPPn36sHr1avr168f+/ftN0nz77bcAnDx50vj5r169mjfffBMHBwcmTpzI3bt3GTduXH4cQq6Ta1iuYSFEwVUggs5bt24Z///g3bs8lZYGMjW3EHnuSVsic8vq1avx9fUlJSWFgQMHMmrUKH744QcSEhIYNGgQQUFBmJmZcenSJYKCgoytCA/atGkTvXr1ws3NDQAbGxvjOltbW9q3bw9AvXr1CAsLy7AcTZs25ZNPPqFLly60aNGCqlWrZpguNjaWN954gy+//JKmTZsSFxfHzp07uXbtmkmac+fOPfbYvb29jcdTr149pk6dCsD27dtp3bo1Xl5eAFhYWFCoUCHjdu+++67x/2vXruXIkSPUqFEDgNTUVLRaLQDR0dH069ePkJAQzM3NuXnzJmfOnKFJkyaUL1+ed955hxYtWvDGG2/g7u7Ozp07CQ8P5/XXXzfmr5QiODj4pfzBev36dY4dO8a2bdsA6Ny5M4MHDyYyMpLSpUsb0504cYJevXqh0WiwsLCgRYsWLFmyhDfffBMnJycaNGhgDDByklzDpuQalmtYiOdBTKIee51Fnu0v14PODz/8kHXr1nH16lWaNWuGnZ0d58+fp3Xr1kyYMCHDP/p5Kv4WTPWCqj2g3Yz8LYsQIt+Zm5vTuXNnRowYwQ8//MDo0aMpVqwYx48fx9zcnE6dOpGYmPjE+ep0OuP/tVotqampGaabNm0aZ86cYdeuXfTu3Zu3336bkSNHmqRJSUmhS5cudO7cmZ49ewKGZydqNBoOHz6MhcWTVSIPly2747seHBKhlGLs2LEmP2LTvf/++7Rt25bffvsNjUZD9erVSUxMRKvVcuDAAfbt20dgYCB169Zl2bJlKKWoUqUKf/311xMdx4vq0qVLFC9e3PhMTI1Gg4eHBxcvXjQJOmvVqsXKlSvp0KEDSUlJrFmzJsNWtqy8CENY5BqWa1gI8XhvzNhLSSdrAvrXzZP95XrT3i+//MLly5dJSUnh6tWrnD9/HoDNmzdnGHCmdx3KMxY2kKYHfe6NRxFCPF927tyJt7c3ALdv38bd3R1zc3OCg4PZvn27MZ2DgwN37941vm/bti2LFy/m6tWrAMTHx5t0ZcuOc+fOUalSJQYPHswHH3zAgQMHHkkzcOBAPDw8GDNmjHGZvb09DRs2NJmQ7d9//+Xy5ctPtP8HtWzZki1bthASEgKAXq83Od4HtWvXjlmzZhl7ruj1eo4fPw4YzmGpUqXQaDT89ddfnDhxAoCYmBiuXbtGw4YN+eKLL2jQoAHHjx+nfv36hIaGsnPnTmP+QUFBJCcnP/WxPO8efiZmRs8wHDVqFCVLlqR27dq0a9eO+vXrP3HwMnToUC5fvmx8ZWeuhYJIrmEDuYaFEBm5cieBi7ficS9s8/jEOaRAdK/NV+ZWoNFCsgSdQrzM0seD6fV6SpcuzezZswHD+KyePXsSEBBA6dKlee2114zb9OzZkz59+rBq1SrjJCRjx46lRYsWaDQaLC0tn/gm2ueff05ISAiWlpbY2Njwv//9z2T9hQsXmD9/Pr6+vsZuezVr1mTu3LkEBAQwdOhQKleuDBhaMWbPnv3UY989PT2ZN28eb731Fnq9Hq1Wi7+/P7Vr134kbc+ePYmOjqZx48ZoNBpSUlLo168f1apVY9KkSQwaNIhJkyZRsWJF6tSpA8Ddu3fp0qULcXFxaDQaypcvT+/evSlUqBAbNmxgxIgRfPrpp+j1ejw8PFi7du1THcfzrmTJksabt+bm5iiluHTpEh4eHibpdDod06dPN75PP98vC7mGHyXXsBAiIwfCogGoV845z/apURndLn2OuLu7P9NdQAC+KwklqkOvdTlTKCFEhlJTUwkJCcHLy8s4VkiInJLZ9ytH6ol81rhxY/r06WPsDTR16tRHWtDu3buHubk5NjY2RERE0LhxY9atW2cypjAwMJDhw4dne/bajM6dXMciN8n3S4jcN3zVCVYfvcyBz5viWkj3+A0eIzv1rMycA4YutsmPf4i0EEIIkR/8/f3x9/fHy8uLSZMmMW/ePABat25tDCDDw8OpWrUqFStWpH379kyfPt0YcCYlJeHu7s6bb77JyZMncXd35/PPP8+34xFCCJF/DoRHU8bFNkcCzuyS7rUAljbSvVYIIUSB5e3t/cgjUsAwP0K6qlWrGsfuPczKyuq5b+0VQgjx7C7diufy7QTequ3x+MQ56KUPOhP1qSSkWGKdHEPexfpCCCGEEEIIkbf2h+f9eE6Q7rXoU9M4fycNfeLzNy28EEIIIYQQQmRX+iRCdcs65el+X/qg08bSnARlhWWqdK8VQgghhBBCvJiUUhwIj6ZcEVuK2udtH8+Xvnut1kxDokaHhUoCpeChZ6EJIYQQQgghckCqHjRmYPaSzUycdL9HpaVt7sQaKUmGfSTHGCZHTf9/Uiwkx95fFsO9u7cZGBdMFRstrJgLLl7Q9MucL08GXvqgEyBZa42ZUqBPMEwqJIR4qZQuXZqNGzfi6+ub5/uePXs2CQkJfPrpp3m+bzA8sD0kJISuXbs+c159+vShZs2aDB48OAdKJkT2yTUs17AooFKS4MoxiNwLkXvg0iHQWkL5ZuD1Ong2BZu87eaZo5JiIOYaxERBzFWIvWr41/iKgthrhsAPAA1Y2oGVnem/GS0zMzdsZwwcH/j/w8vS9NkqbiGgtzlwG7itgbKNc+W0ZESCTkBvZg2pgD5egk4hRI5KSUnB3DzzP7Xvv/9+vu4/KCiIjRs35sgPViFeRHINi3x159L9gG0vRJ0Arfn9wMT+MUGLvaFVTaU+EKjcbwUzBi3pLWFx/63XaMCpLDiXB2dPcPE0/KsrlL3ypiTDlaOmQWZKgmGduTWUrG3Y1+nfDC+NFjzqglcr8H4dXMrn3rl8EkmxDwWRUabBZPry5CzmhLF2BHs3cK8F9q6GZQ8HjnE34VaE4X1aymMK9VDAausCjqUz/+yNy+5/VyxtmfjnJdadvceWka/j4uiYpz08JegEUrT3g87kWMMHKIQQQGhoKJ988gnXr18nOTmZgQMHMmjQIADeeecdzp07R3JyMh4eHsyfP5+iRYsSGBjIJ598QqNGjTh8+DBDhw5l06ZN2NjYEBISwsWLF/H19WX58uVYWloyfvx4YmNjmTp1KgsXLmTZsmU4OTlx+vRprKysWLlyJWXLlgVgzJgxrFixAmdnZxo1asSuXbuMz2h8UOPGjXnllVc4cOAAAFu3buWNN94gOjqahIQEqlatypw5c4iNjeXLL7/k3r17VK1albp16zJ79mwOHz7MqFGjuHfvHmlpaYwZM4bOnTs/sp8rV67Qq1cvbty4QZkyZUhNTTWui4mJYejQoZw4cYLExETq16/Pzz//jIWFBVeuXOHjjz82Pt6jffv2fP311/z666/89NNPJCcno5Ti22+/pXXr1qxatYq5c+eydetWwPDw+LJly7JlyxYqVqyYsx+6eKHINSzX8FO5FQHHl0CljuBaOW/3fffyfwFb5F64HfnfOgd3w7/J4YaARaVmmMUT01r9F6CkpRqC24fZFn0gCE0PSMuDQ3G4euq/8l48+GiQWbohlGkIxauDuaVhXcw1CN0KwX9A+C648Dds/wKcyhmCT69WhmBUa5Ezx/iw+FsQfd7wuhkKdy89EFxeMwTkmdEVNgSR7jUNQaW9K9i5Gv41vi8GFk82ZjItOZENR0KY++dJ9AkxaEnl+x71qVi6uOGzsbABs6efjkcpxYbLN3Es5oCLU963LkvQCaSa20Ay8qxOIfLDr93hdkTu5O1YBnosf6pNU1NT6dGjB0uWLMHHx4f4+Hjq1q1L3bp1qV69Oj/++CMuLoabVJMmTWLChAnMnDkTgJMnTzJz5kxmzJgBwKZNmwgKCmLHjh1YWlrSqFEjfvvtN956661H9nvw4EFOnDhBqVKl+Oyzz5g8eTL+/v5s2LCBjRs3cuLECaytrenSpUuW5Q8KCuKPP/7AwsICpRS//vorzs7OKKUYNGgQs2bNYvjw4UyYMIGNGzeyevVqAO7cucPAgQPZtGkTbm5u3Lx5kxo1avDKK6/g6upqso8hQ4bQqFEjxo0bR3h4OH5+frRq1QqAYcOG0ahRI+bMmYNSigEDBjBz5kw+/fRT3nnnHVq3bm3c540bNwBo2bIlb731FhqNhsjISOrXr8+FCxfo1KkTI0aMIDQ0lPLly7N27Vo8PT0L5o/Vl1FuXsPw1NexXMNyDT+V+FuwtDPcCoM9Pxi6H9b7yNANNDdahbIKMp3KQvVehqCt1CtQqMR/65QydF01tmDGPtBq+cBYPjNzYyuXsRXs4W6cDwd2yfFwK/x+UBYK0WGGwOz6Gbi4L/NjMddByTqG8pZuACWqg7lVxmntixmOrXovw/C2iD0QssUQhO6faXjpCoFnc8NnYOP8UMud7X8tvJkFYvrEjI8j+jwk3Ho0va7w/ZbJGhkEk64PBJPWmZ+Dp3T233t8ue40Ry7cxtm2GF1frcn/AsNYe8maipVdH59BNkTcjOPavSRaVcqZ/J6UBJ1Amvn9LrV6CTqFEAbBwcGcOXOG7t27G5fFxMRw9uxZqlevTkBAAEuWLCEpKYmEhASTH3NeXl40aNDAJL9OnTphbW2oqGrXrk1YWFiG+23QoAGlSpUCoF69evz8888A7Nq1i65du2JrawtA7969+frrrzMtf8+ePbGwMPyQUEoxffp0Nm3aREpKCnfv3qVRo0YZbrdv3z7Cw8N5/fXXjcuUUgQHBz/yg3XXrl3GH+Vly5aladOmxnVr167lwIED/PDDDwAkJCRgaWlJbGws+/btY/v27ca0RYoUASAiIoK3336by5cvY25uzs2bN7lw4QKenp7GH9nTp09n5syZDBkyJNNjFwLkGpZr+Cmk6mFVH0PA2WAo3LkAZ9ZCeCAUrQj1PoTKb2YeSGWHPgEi/oLgLYYWvuwGmQ/TaAwtaRa6nO+lZ2kDrr6G14OUgvjo/1oHo88bWgiL+NwPMms83bmxsAavFobXG9Pg6klD8BmyBU6vNryy3D49oLb9r0Uw5l9D12SUaVobFyjiDc7lDK216a22jqVyJZh8nLsJeqZvD2Hx/kgAetcrxdDm3jhYm7Pu+BU2n4ri89d90OTADY/8ej5nOgk6gbT0L1lyXP4WRIiX0VO2ROY2pRQuLi4EBQU9sm7v3r3MnDmTffv2UaRIEdavX8+ECROM6+3s7B7ZRqf7r5uNVqslJSXjsRuZpVNKPVGl82AZfv31V3bv3s1ff/2Fvb09M2bM4K+//spwO6UUVapUyXR9dimlWLt2rbFbYbrY2MzHv3Tv3p2pU6fSoUMHAJycnEhMTARgwIAB+Pr68tZbbxEeHk67du2eqXwiB8k1nK10cg0/B9fwH59BxG6o+a5hRk+NBpqNhwOz4dhiWPch7JgAtd8zpMnuBDgxVyFkK4T8YQhg0xs5HEtDtZ7/tQxmFWQWBBqNIcC1dTF0fc2tfbj5GV6NR8G9KLhy5KFxqemzsWYwLjXxrqGLrF0xqNj+v27Azp6GQNPaMXfK/YSUUvx+7ArfbfmHm7HJVPcozIT2vviW+G/s7OuV3Zi3N4LTV+5R2T2bY2qzcCDc0Lpbp0z+BJ0v/XM6AcMdEkCfmEX/bSHES8Xb2xsbGxsWL15sXHb+/Hlu3brF7du3cXBwwMnJieTkZPz9/XO9PE2aNGHVqlXEx8eTlpbGkiVLsr3t7du3cXZ2xt7enpiYGBYuXGhc5+DgwN27d43v69evT2hoKDt37jQuCwoKIjk5+ZF8X3vtNebPnw9AZGQkO3bsMK5r164dkyZNMv7gvn37NufPn8fOzo4GDRowffp0Y9r0rnm3b9+mdOnSACxdupTbt28b0zg6OtK2bVs6d+7M+++/j1b7kk23L56YXMNyDT+RQ3Pg8Fwo0when/JfV9rCHtDqWxh6Bpp/beiuuvNrmF4JNo8wdN98mFIQdRJ2T4H/awI/eMOGIYbAs3g1Qz6Dj8DHJ6D9TPDrVvADzvzi4AYV2kLVt6D2AGg41HBD4PXJ0OEX6LoYev4O/bbCB3/DJydhxHnD/7sugqZfgF93w/jLAhJw/hN1j67++xm26gRpCqZ0qcLq9+ubBJwAre93q918OuqZ96mUYn9YND6u9jjaWj5zfk9Dgk4wzlirj89iBiohxAutWbNmuLu7G19Xr15lw4YNrFy5kipVqlCpUiX69+9PQkICr7/+Op6envj4+NCyZUuqVq2a6+Vr164dLVu2xM/PjyZNmlCuXDkKFcrenc9evXoRGxtLxYoV6dSpEw0bNjSua9q0KXFxcfj5+fH+++/j6OjIhg0b+Prrr/Hz86NixYp89tlnpKWlPZLvTz/9RGBgIFWqVGH48OE0a9bMuO7HH3/E3NycqlWrUqVKFZo1a0ZkZCQAS5Ys4cCBA1SqVAk/Pz/jOLqffvqJjh070qBBA06cOIGHh4fJ/gYMGMCNGzfo37//k54+8RKQa1iu4acWtgu2jDJ0b31zUcaT1+gKwStDDIFipzmGlrND/wczqsOKd+DCPgjZBhs/NQSk/g1h1zeGcYSVOhm2GXEe+m425FNQZmkVeeZeop6vNpyhzc97OXrhNj3rlmLXsMZ0rVkSM7NHe0FUK+lIMQcrtpyKQimVQY7ZF3YjlpuxSfnWtRZAo571KPKZu7s7ly9ffqY8ApbO4+3zQ7nT7AcKNyhgfwiFeIGkpqYSEhKCl5dXwbvL/RyIiYnB3t6etLQ0+vfvT/HixZk4cWJ+FyvPTJkyheDgYObNm5fh+sy+XzlRT7ysMjp3ch0/PbmGs76GIR++XzfPw9zXDEP/+v8JRbyyt51ShrGZ+2dC6DbTdY5l/puBtVT93JuBVTw3Np+K4st1Z7gZm0Q1j8J8/VBX2syMW3eaRfsvsOXjhlRwc3jq/S/ZH8kX687wfz1r0CIXJhLKTj0rYzoBM6v73WsTpKVTCFFw9erVi8jISBISEqhevTojR47M7yLlmUqVKqHRaPjjjz/yuyhCPDW5hgvYNZxwG5Z1M4wTfHt19gNOMHS/Lfuq4XX9HAQtNUxS4/06uHjl6fMPRcF2/V4ig389RmEbS6Z0rkKXGu4Ztmxm5PXKboag81TUMwWdB8JvodHk33hOkKATAK3OEHSmJkrQKYQouNasWZPfRcg3Z86cye8iCPHM5BouQFJTDDPVRp+HVpMNj0R5WkV9oMXL02ItnsyOc9dJUzCubUXaV32ysbu1SjvhYmfJ5tNXGdrC+6n2r5TiQHg0Fd0cKGSTf63uMqYTMLcyzBCXmiRBpxBCCCFEgXb5CCTcebY8tn5umEm2Rh+oMzAnSiVEhrafvYa5mYbGXkWfeFutmYaWlVw5fz2W0GtPN+FpyLVYouOSqVc2/1o5QYJOACys7QFIk0emCJGr0h8X8JwPJRcFVPr3KieeZyYyJ9exyE2PvY53fQdzmxom6/njc7h94cl3cniuYRKg0g2h9VTpCityTVxSCnvP36R2GaenbmVsXdkNgM2nrj7V9gfy+fmc6aR7LWBhbWjpVEkSdAqRm8zMzLCwsCA6OhpnZ2cJDkSOUUoRHR2NhYUFZmZyPzU3yXUscoter+fatWvodLqMr+Nd38HuSVDM1zCRz4FZcHA2VOwA9QdDiRqP30l4IGweaZjsp+timeTnBRKfnMLZf+8Rej2WphWKUtRe9/iNctme0Jskp6TRvGKxp86jThknHG0s2HI6io+bPfmsx/vDojHTQK0y2XyubC6RoBPQ2RhaOtFL0ClEbvPw8ODixYvcunUrv4siXjAWFhaPPKJB5A65jkVu0Gg0FC5cmKJFM+iGmB5wulWFXmtBVxjCdhpmjz3zu+HlUR/qf2SYNTajoDU6DFb2Bktb6LECbPL3R7h4egnJqZyNusvJy3c5deUup6/c5fz1WNLud8BoVcmV2T2zcRMil20/ew2AZhWePug015rRoqIrK45cIvxGLGWL2GV727Q0xYGIaHxLFMJBl783WCToBHQ6HclKiyY5Pr+LIsQLz9LSEk9PT9LS0qR7nsgxGo1GWjjzkFzHIqdpNBrj6xEPB5zWjoblnk0Nr6unYf8vcGoVLH/L8AzNuoOgag+wsDakTbgDv3aDpHvQYxUUebpJWUTeSw8wT12+y8kMAkwAd0drWlZyxbdEIXYH32Dr2atciI6jlLNtvpU7JTWNneeu4eNqT0knm2fK6/XKhqBzy+mrfNjEM9vbnbsaw514Pd1q5m/XWpCgEwBbS3Pi0aFJkaBTiLwiAYIQzz+5jkWuC5yUccD5IFdf6Pg/aPolHPKHI/Nh01DY9Q3U6g81+sK6QRAdCi2/g/LN8v44BJE34xi6MojTV+490Xb6tDQevLdVovB/AWbl+y9HW0vj+uoejrw15wDz9kYwob1vThX/iR27eIfb8Xp61i31zHnVL+eCg86cLaejnijoTB/PWTefx3OCBJ0A2FhpiccKbUpCfhdFCCGEEEKAIeAM/A7c/DIPOB/k4AbNxkPD4RAUYGj93D0Zdk8BFFTvBXU/yIOCi4dtOhnFqN9OEpecQsPyRbDUZn8suJWFlopuDlQuUQjfEoVweiDAzEjdsk5ULlGIlUcu8Ukzr8emzy3bzxom/mle0fWZ87I0N6N5RVd+O3aZi9HxeDhnr+V0f3g0WjMNtUrnf1dyCToBG0st8coKe2npFEIIIYTIfyYB57rHB5wPsrIzPAalVn/4Z4NhsiEbZ2j9g8xUm4VLt+I5EB5NW7/i6Cy0OZJnoj6Vbzf/w+L9F3Cxs2T2O3VoUN4lR/LOjEajYUCjsgxZdpylBy4wpOmTT77zrJRSbD97DVcHHb4lHHIkz9aVDUHnH2eieK9RucemT01THAyPpnKJQthZ5X/IJ/1iABtLc+KxwiJVWjqFEEIIIfJV4OSnDzgfZKaFSh3g3T+gewCY50+L1/Piy3WnGbH6JE2mBrL66GXS0p5tvPaF6Di6zN7H4v0XqFvWic1DGuZ6wJmuta8rJQpbs2hfJIn61DzZ54PCbsQSGR1Ps4pFc2yG7wblXbCzMs/2o1P+ibrHvcSUfH9USjoJOrnf0okOizQJOoUQQggh8k3gZAj81hBw9sxGl1qRI+7EJ7Mn9CalnW1I0KcyfNUJ2vy8l72hN58qv82nomgzYy9n/r3HkKblCehfl6IOefcIE3OtGe82KEN0XDJrjl/Js/2m23Z/1tqc6FqbzspcS9MKRQm6dIcrdx4fsxjHc5aVoLPAsNCakYgOi7TE/C6KEEIIIcTLafcU04BTHmmSZ7aduUZKmmJQE092j2jCwEZlOX8jlnfmHaT3/EOcu5q9yX+SUlIZt+40gwKOYWluxuJ3Wqj80wAAIABJREFUazO0uRdas7zv1tytVknsdebM2RP+zK22T2r72WvYWZlTt2zOfodf93UD4I/Tj2/t3B8WjbmZhpqlCsaNGwk670sy02GlkiAt75vghRBCCCFearunGGabda0iAWc+2HDyXyy0GlpWdKWQtQWft67AzmGv0rFaCXaH3KD1T3sYufoEV+9m3kBzITqOLv/bz6L9F6hTxonNHzekYfkieXgUpuyszHmnbinCb8Sx49z1PNvv9ZhEgi7d4VWvIliZ58zY2HSNvYtgY6lly6moLNOlpKZxKOIWfiULY1sAxnOCBJ1GerP7z3HSy2RCQgghhBB55sGAs9c6CTjzWHRsEvvComlYvgiFbCyMy90dbZjerSobBjegThlnVh65TOOpu/hhWzCxSSkmeWy535329L93GfKaJwH961AsD7vTZqZP/dJYaDXM+Ss8z/a585/rKAXNKxbL8bx1Flqa+BTl6MXbXLuX+Q2As1H3iElKoV4B6VoLEnQa6bX3g85kCTqFEEIIIVAKLh8FfS4NP4q/BeuHSMCZz7aeuUZqmqJNFbcM11d2L8SvA+owv09NSjra8PPO8zT+fhdLDlwgPjmF8evP8MH97rSL+tZmaAtvzLUFI8Qo5qCjfdUSHIq8xfGLt/Nkn9vPXkNrpqGxd+608rb2dUMp2Hom8y62+8MK1nhOkKDTKNU8vaUzLn8LIoQQQghREOz7Gea+Br/UgpOrIC0tZ/JNS4Oji+DnGnBsEZRuKAFnPtp48l8stWY0y6JlTqPR8JpPMbZ83JDvOlVGo9HwxdrTVP96Owv3RVL7fnfaRl751502MwMalgVg7p6IXN9XfHIKe8/fpHZpJwrb5M5syY29i2BlbsbmLLrY7g+PxkKroUYBGc8JEnQapWrvP2Q1WYJOIYQQQrzk/g2CHRPAwR0S78Lv/Q0BaMSeZ8z3OMxrBhuGgNYCOs2F3hv+n707j4+yPPc//pl9si9AAiEJYQs7JQiIoCCCG25FtLVHW2ytYqs/e4pVe2y1drMeTw89blVqobbaii3UjeKuqFCUfZMdAplASAjZM0lme35/TBJZEsg2mUnyfb9evJJMnnnmiqLJN/d1X7cCZ5gcr6zjs4MnmD6sD/FO2zmvt1rMfGNSJqt+dDE/nJVNrMPG3TOG8LcIaadtyrC+cUzP7sNbOwrIOxHajsZP9xVT5wuEpLW2QYzDysXD+rAut4TjlXVnfN7nD7A+t4ScjCSi7B27p7Q9FDrrGbaG0Kn2WhEREenBPNWw/LvB97/xN7hnC0z+PhzbAX++Gv72dSja3bp7uktgxQ/hDzOCgXbyXXD3Bhh7I3TQOYbSem/vKCBg0GxrbXNiHFZ+MGsoG346ix9dHjnttM2ZP20QAQOWrAntaud7jUelhC50Aswe04+AAe/uPLPFdvuRcqo9fiZHyPmcDSL7b0gnCtSHTn+dVjpFRESkB3vnJ3BiH8x8OHh8SXQyXPEbuHs9jLoe9r4Nz14Ab/4AKgvPfq9AADb9JdhKu2EJDJgCd66GKx4FZ3znfD3SrDe3FeCwmpk5IrQhKdwuGNyLUWnxvLLeRWm1JySv4Q8YfLi7iOF948hIjg7JazS4ZHgKdouZt7afGTrXNp7PGVndAwqdDewxAHhqKsNciIiIiEiY7FoBG/8EA6fDBXef+rnkgXDjn+C7H0DGZNj4AjyZA6sea3p70tHNsPhSeOP/1bfSPg+3/gtSR3bKlyJnV1hRy/pDJVwyPIXYCDlWI1RMJhN3TBtEjdfPXz8/HJLX2JRXSkm1J+SrnABxThvTsnuz9uCJM0L02gMnsFvNjM+MnP2coNDZyNQQOt0KnSIiItIDVRQEA2JUEsx5DszN/JiYPgG+vRJu+hvEp8Gq3wTD58YXwO+rb6VdUN9KuznYmnv3ehj7tS7RSru3sJLb/7KBe/++lf9+ezcvrMnlre0FbDxcgqvETZ2ve5zpvnJ7AYYBV49NC3cpnWL2mH6kJTh54d+HqfV2/L/DhtbaWZ20anzl6H74A0bj6wJ4/QE2HCplfGYiTlvk7OcE6N6/1mgFsz24DO6trQpzJSIiIiKdLBCA1+6EmhL4+kvBMHk2JhMMvwqGXhacQLvqsWC77drfg7sY3Ccgcwpc9VtIHdU5X0MHeXld3ik/yDclMdpGapyTlHgHKXFO+iY4uH58OoP7xHZSle23YlsBUTYLM4ZH3sTZULBZzHznwoH86l+7eH3LEb4+MbND7//+zkJS4x2M6Z/QofdtzqwRqVjNJlbuKOBrEzMA2JZfRo3XH1FHpTRQ6KxndgT/J+GtUegUERGRHuazZ+DgKhg/D0Zc0/LnWWww8bsw9uuw5gn499PgiIM5f+gyK5un23ColOQYOx/fdzHHK+sorKijqLKWooo6CitqKayso6iilqLKOjYeLsXtCa6a/WXtYf74rQmcH4E/8J/uaFkNGw+XcvXYfkTbe04c+PrEDJ54fx/Pf5rLjedlYDZ3zN/P/UVVHCyu5ubzMzvsnueSEG1j6pDerNlfTLnbS0K0rfF8zgsi8O9gz/lbdg5WZzB0+j0KnSIiItKDFGyD938OvYYEBwa1hSMOLvlpsJXW6micldHVVNX5+OJoObNGpBLntBHntDHoLKuXhmFQVedjXW4JP1i6hW8uWceTN43jitGtmwbb2RrOeOwprbUN4pw2/mNyJos+PshHe4o6bIDS+7vqW2s7YT/nyWaP6cvHe4/z/q5C5p6XztqDJ3BYzYzLTOzUOlpCezrrNYTOgNprRUREpKfwuGH5bcH35/6x/WExOrnLBk6ALXllBAyYNLBlkz9NJhNxThszR6Sy9I7JxDttfO+vm3jxs9AMq+kob24rIMZu4eJhPaO19mTfnjIQq9nEHz452GH3fG9nITF2C1M6+ZiSS0f2xWI28daOY9T5/Gw8XMqErCQc1sjazwkKnY1s0cHQaejIFBEREekp3v0JFO8NrlKm5YS7mrBbf6gEgAlZrT9uYnT/BP75vSkMSI7modd2sPDdPRiG0dEltpurxM1WVxmXjkyNuGEznaFvgpNrx6XxeW4JW11l7b7f8co6NuWVMi27T6eHveQYOxcM6sUn+47z7/0nqPUGmDww8lprQaGzkd0ZB4DhdYe5EhEREZFOsHtl8OzMgdNgyj3hriYirD9UgtNmZlRa284QzewVzbLvTWFsegJPfrifB1/djs8f6OAq2+dfPbS19mS3XzQIgOc/bf9q50e7izAMOuWolKZcMbovHl+Ax97aDQTPJI1ECp31HPUrnXgUOkVERKSbqzwGb9wNzkT46lmOR+lBvP4Am/PKyMlIwmZp+z+P3rEOXr59MtOy+/DyOhd3vrQpJEd0tNWKbUeJc1q5KLt3uEsJmxH94rloaG9Wbi/AVdK+n/3f3VmIxWzikuEpHVRd61w+qi8mE+wprCTKZmFseuTt5wSFzkYxTge1hg2zV+21IiIi0o0FAvDa94LHmlz7JCT0D3dFEWHn0QpqvH4mZiW1+14xDit//NYE5uT05/1dhdz8x88pc3s6oMr2OVRczY4jFVw2sm9E7vvrTPOnDSZgwJI1uW2+R43Hz+r9x5kwIInEaHsHVtdyfeIcTKpvB5+QlYTdGpnxLjKrCoNouwU3Dsw+rXSKiIhIN/b5s3DgQ8j5Joy8LtzVRIz27Odsit1q5n9v/Ap3TBvExsOl3PjcWo6W1XTIvduqsbX2K5E9XbczTB3SixH94nllvYtyt7dN91i9v5habyBsrbUNZo8J/vuMxPM5Gyh01ot2WHHjxOwL7/8MREREREKmYBu8/wgkD4YrHgt3NRFlw6FSzCYYP6D9K50NzGYTD84ewU+vGsG+oirmPvtv9hZWdtj9W+vNrUdJiLIxdXDPba1tYDKZuGPaQNwePy993rZpw+/tPAaEbz9ngxsnpHPPzKHcfH5mWOs4G4XOetE2C27DgdWv0CkiIiLdkMcNy78LRgDmPg+O5s+f7GkMw2DD4RJGpsUT6+j4Y+y/e9EgnrhpHMVVddzw7L/ZUL+q2pn2F1Wx+1glV4zqG7EtmJ3t6rFp9Etw8tSH+1i8Ohd/oOXThv0Bgw92FZGdGsuAXuE9JijabmXBpdlha/FtCf2NqxftCLbX2vxqrxUREZFuprY8GDiL98CMn0D/88JdUUQ5dMJNcZWHCQM6prW2KdeN68+SWyfiDxjc/MfPeW9nYcheqyn/2qbW2tPZLGae/o/xpMQ5+eWKndz43L/ZX9SylegtrlJOVHvCvsrZVSh01rNbzNTixB6oDXcpIiIiIh3n6BZYNB32/AtGXQ9TfxDuiiLO+tzgyuPEDtrP2ZyLhvZh6R0XEOuw8p9LN+PtxONUVmw72niuo3zpvAFJvP2fF3HbhQPZ7Cpj9hOrefrDfef8d/Nu/S8NLh3ZtzPK7PIUOuuZTCZqzQqdIiIi0k0YBqx7HhZfChVH4Mr/gRuWgLlnTy1typdDhDpuP2dzxqQncPP5mVR7/OwqqAj56wHsLaxkX1EVV4zui7Udx8F0V9F2Kw9dPZLl35vCgF7R/PbdvVz79Bp2HClv9jnv7SwkJc7B2P4JnVhp1xXyv3X33HMPWVlZmEwmduzY0eQ1r7zyCjk5OYwePZoxY8bw1FNPhbqsJnnNUdjwgr9tE6xEREREIkJtBSz7Nqz8EcT1g++8A+ffASZTuCuLSBsOl5KZHE1qvLNTXi+nfljR5ryyTnm9FVuPAnD1WLXWns34zCRW3HMh91wyhH2FlVz3zBoee2v3GeesHjhexcHj1cwckYrZrP+mWiLkofOGG25g9erVDBgwoNlr0tPTeeutt9ixYwerV6/miSeeYM2aNaEu7Qxec1TwHY/O6hQREZEuqmAr/GE6fPEqjLgW5n8C/ceHu6qIdbyyjtzi6k5Z5WwwLj0RgM15pSF/LcMwWLGtgN6xDs4fqNbac3FYLSy4bBhv3H0hI/vF89zHB5j9xKesy/1y+NP79a21l2k/Z4uFPHROmzaN9PT0s14zdepU+vYN9kMnJCQwfPhwcnPbflBrW/ks9aHTq2FCIiIi0sUYBqxfDH+8FMpcwXbar/0FohLDXVlE23g4GCYmhXg/58mSYuwM6h3DZlfoVzp3FVRysLia2WP6YtGqXIuNTIvn1e9P4cdXDudIWQ1fW7SWh1/fQVWdj/d2FhJtt3DBYIX4loq4pu6dO3eydu1aLrnkkk5/bb81OviOR6FTREREupDaClj2HfjXAojrC7e9q3baFlp/KLjaOKETQyfAuMxEDp9wc6KqLqSvs2JbQ2ttWkhfpzuyWszcOX0wb/3gIiZlJfOXtYe5/HefsDGvlGlD++C0aX90S0VU6MzPz+e6667jueeeIy2t6f8wFi5cSHp6euOfqqqqDnt9v7VhpVPttSIiEjn27dvHlClTyM7OZtKkSezcufOMa6qrq/n2t7/NmDFjGDZsGD/+8Y8xjC/PnFu8eDFDhw5l8ODB3HHHHfh8vs78EiSUCrbVt9P+E0Zc023baQ3DYFdBBX/45ACFFR03+HHDoRKSom0M7tO5Zy3mZAbbebeEcLWzobU2Nd7BhAGd1z7c3QzqE8vSOybzy+tGUeb2YBjoqJRWipjQefToUWbNmsVPf/pTbrzxxmavW7BgAfn5+Y1/YmM77mBjwxZc6QzUdlyQFRERaa/58+dzxx13sHfvXu6//35uu+22M6559NFHAdi2bRs7duxg8+bNLFu2DIDc3FweeughVq9ezf79+zl27BiLFy/u1K9BQsAwYMMS+OOs+nbax+FrL3a7dtq8E26e/nAfl//fJ1z5xKc8unI3v31nT4fcu7rOx46jFUzISsbUyavCORnBf0+bQrivc8eRCvJK3Mwe008Db9rJbDbxzQuyeHfBdH5z/RiuG6eV49aIiNBZUFDAzJkzeeCBB5g3b17Y6jBswd9weWoUOkVEJDIUFRWxadMmbrnlFgDmzp1Lbm4uhw4dOuW6rVu3cuWVV2IymbDZbFx22WW8+OKLACxbtow5c+aQmpqKyWTizjvv5OWXX+7sL0U6it8HBz6EpTfDih9CXCrc9g6cP7/D22lPXi3vTEWVtfxpTS5zfr+Gaf/zEb99dy/HK+u4+fxMhqbEsnJ7AW5P+1frt7jK8AcMJnbiEKEGw/vGEWWzhHSCrVprO17/xCi+MSlTR8+0Usj/ad11112kp6eTn5/PrFmzGDJkCACzZ89mw4YNADz88MPk5eXxxBNPMG7cOMaNG8ef/vSnUJd2BpM9GDrraio7/bVFRESa4nK5SEtLw2q1AsFzpTMzM8nLyzvluokTJ/L3v/8dj8dDZWUlr776amMwzcvLO2WKfFZW1hnPh9BuYZF2MgxwrYe3HoCFI+DFObDnX/XTaT+F/ud1+EtuOFTC8IfeZuX2gg6/d1Mqar38fYOLby7+nMmPfsDP39zJnmOVfHVcGn+6dSLrfjKLX88Z03jG5btfFLb7Nb88n7Nz93NCcL/g2PQEttYH347W0FqbluBsXFUVCRdrqF/gmWee4Zlnnjnj8ZUrVza+//zzz/P888+HupRzcwTba71a6RQRkQhyettfU6tPDzzwAP/1X//FpEmTSEpKYsqUKXzwwQdN3qO51asFCxawYMGCxo/PNX1eOkHRbtj+D9ixDEoPBR9LHgwTvgNjboDeQ0PysoZh8OjKXdT5Avz8zS+Ynt2HGEfH/9jo8wd4b2chr285yod7ivD4AtgsJi4Znsp149KYNSKVKPupw1quHdefX6/cxfJN+Xw1p3+7Xn/DoVKcNjOj0xLadZ+2yslM4vPcEvYVVTK8b3yH3nuLq4wjZTXcftFAtdZK2IU8dHYlFntwf6ivViudIiISGTIyMsjPz8fn82G1WjEMA5fLRWZm5inXOZ1Ofve73zV+/NhjjzFy5EgAMjMzT2nHPXz48BnPlwhSlgc7lsP2ZVC4I/hYXD+44G4YPRfSckI+lfaDXUVsyitjQK9oDp9w8/RH+3ngiuEd/jo/eXUHr2xwYTLB5IG9uG5cGleM7ktitL3Z5yTH2JkxLIX3dhVSUF5Dv4SoNr22zx9gU14p4zISsVvD0yqZk9lwXmdZh4fOFduCK9RqrZVIoGbkk1icwfZanwYJiYhIhEhJSSEnJ4eXXnoJgOXLl5OVlUVWVtYp11VUVOB2B4/8ys3N5dlnn+Xee+8FgvtAX331VQoLCzEMg+eee46bbrqpU78OOYdAIDgUaMkV8H9j4P1HoNwF478F896EH34Bl/86OJU2xIEzEDD47bt7cNrMLL1jMiP6xbP401xyizt2uv+n+47zygYXUwb3Yu2PZ/LyHZO5aVLmWQNng+vHp2MY8Nrmo21+/V0Flbg9fiaGobW2wZehs2OHCRmGwcrtBWQkRzE2PTyruCInU+g8idUZB4C/TkemiIhI5Fi0aBGLFi0iOzubxx57rHHy7MnzEQ4ePMi4ceMYOXIk1113Hb/73e8YN24cAIMGDeLnP/85U6dOZfDgwaSkpDQ5AVfCaMtfg0OBjm6BUdfDTS/Dj/bBtU/BwGlg7rzzAN/cdpTdxyqZNyWLfglR/PzaUXj8AX654syjetrK7fHxX//cTrTdwn/PHUvfBGernn/J8BSSom0s35Tf5mFH4dzP2SAlzkl6UhSbOniY0O5jlRSU13LZyL6dPpVXpClqrz2JzRlsrw0odIqISAQZNmwYa9euPePxk+cjjBs3jr179zZ7j9tvv53bb789JPVJB8j9JPj2B1sgrm/YyvD6Ayx8by9xDivfmz4YgEkDk7luXFpw3+XuQi4Z3v7zCf/33b3kl9bw8NUjyUiObvXz7VYz134ljT+vPcy2/HK+0oZBOesPlWA2wfjM8A7ZyclM4s2tRymv8ZIQZeuQe3689zgAFw/r0yH3E2kvrXSexB4VDJ2GR6FTREREOpHrc+g1NKyBE+DvG1wcPuHmjmmDTmlz/a8rRxBtt/CLN3dS5/O36zW2uMr405pcxmUkMm9KVpvvc/344KCrf27Kb/VzDcNg/aFShveNJ87ZMUGvrRomy251ddxq58d7juO0mcPaOixyMoXOkzhigu21KHSKiIhIZ6k8BmWHIeP8sJZR6/Xz5Af76B1r5zsXDjzlc30TnNw1YwiHTrhZsvpQm1/D4wvwwLJtWMwmHr9hLJZ2TFUdm57AkJRY3th6FI8v0KrnHj7hpriqjkkDwx/KTh4m1BGq6nxsOFzCBYN64bR1Xlu2yNkodJ7EERVDwDBh8rrDXYqIiIj0FK51wbcZk1r1tG35ZRRX1XVYGX9Ze4jCijrumjGkyeNRvnvRQAb0iuapD/dRWFHbptd47uMD7Cms5PsXDyE7Na5d9ZpMJuaOT6fU7eXD3UWteu6X+zmT2lVDRxiZFo/dYmazq2OGCa09cAKv32B6tlprJXIodJ4kxmGnBrtCp4iIiHQe1+fBt61Y6dzqKuO6Z9Yw5/drKKn2tLuEylovv191gP6JUfzH+U0fp+OwWnj46pG4PX5+s3JXq19jf1ElT3+4n6EpsXx/xuD2lgzAV3PSMJla32K74VAw4E0YEP6VTofVwqj+8WzOK2vzUKSTfbw3GMCnD0tp971EOopC50mi7RbcOLD4FDpFRESkk7jWgTMBeme36HJ/wOCh14Pnd7pKarjzpY2tbi893fOf5lLm9vKDWUNxWJtvyZw5IpUZw/rw2pajbKhfLWyJQMDggeXb8QYCPDZ37FlfozX6JURx4ZDefLSnqFXhe/3hEjKSo1o9NTdUxmcmUV7j5WA7j6UxDINVe46TmRxNVq/WD2gSCRWFzpNE2y3UGA4s/ppwlyIiIiI9gbcWCrYEVznNLfux7OV1eWzLL+e7Fw7k1ilZrMst4eHXd7R5lexEVR2LPz3I4D4xXJ/T/5zXP3T1SGwWEz974wv8gZa95oufHWbj4VLmXZDFeQM6tqV17vh0vH6DN7YcadH1J6rqOHi8mokRsMrZoKP2deYWV5NfWsP07D46KkUiikLnSaLtVqpxYlXoFBERkc5QsBX8nhbv5zxRVcf/vLOH1HgHP5iVzU+vGsG07D4sXe9iyZpDbSrh96sOUO3xc+9lw7Bazv2j4aA+sXznwoF8cbSCpevzznn9kbIaHn97N/0To7jv8mFtqvFsLh/Vlxi7hX9ublnoXN/QWhtBk11zMoNBfHNe+/Z1NhyVov2cEmkUOk/itJmpwYFdoVNEREQ6g+uz4NsW7ud87K3dlNd4eejqkcQ6rFgtZp76Rg6D+sTw63/t5KM9rRuoc7Sshhc/O8zo/vFcMarlx7X8v0uGkhLn4Lfv7KHM3Xxbq2EY/OTV7VR7/Dx6/ZgmBxS1V5Tdwuwx/diWX86+wspzXt/QFjwxAoYINUhLcJIS52j3SufHe49js5i4YHCvDqpMpGModJ7EZDJRZ3JiDyh0ioiISCdwrQOTBdLGn/PSDYdK+MfGfC4c0purxvRrfDwhysbieROJc9q452+b2V907uDV4MkP9uHxBbjv8uGYW3F8SazDyn/NHk6p28vv3tvb7HWvbznKqj3HuT6nf0hX3+aeFzyzc1kLBgqtP1xKUrSNISmxIauntUwmEzmZiew+VoHb42vTPWq9fj47eIKJWckhCfci7aHQeZo6cxQOoxY6YHqYiIiISLMMIzi5tu9ocJw9APn8AX762g5sFhM/v27UGfv1BvaO4dmbx1Pj9XPbnzdQ2oKhOgePV/GPjflMGpjMtKG9W13+V8f157wBSbz42WF2FVSc8fkTVXX8/M0v6BVj56GrR7b6/q0xKSuZ9KQoXtt85Kz7TN0eH18cKee8AckRt+cxJzOJgAFbXeVtev663BJqvQG11kpEUug8jdcchYVAcH+FiIiISKiU5kL18Ra11v5l7WF2H6vkjmmDGNyn6YA6ZUhvHrl2FIdPuPneX8890Xbhe3vxBwweuGJYmwKYyWTi59eOwgAeeeOLMwYZ/WLFTkrdXn527SiSYuytvn9rmM0mrs/pT2FFHWv2Fzd73RZXGb6AEVGttQ3GN+zrbON5nY37OYcpdErkUeg8jc8SFXzH076R1SIiIiJn5VoXfHuO0FlUUcvv3ttL/8Qo7p4x9KzX3jJ5APMuGMBnB0v4WRNBsMGOI+Ws2FbAzOEpnNeOKa6j+ydw08RMPs8t4V/bCxof/3B3Ia9vOcrM4SlcM7bfWe7Qca4fH2yxXX6WFtsNEThEqMGY/glYzKY27+v8eO9xUuMdDEuN6+DKRNpPofM0PqtCp4iIiHQC1+fBt+eYXPvoyl1U1vn42TUjibKf+3zLh64eyUVDe/Pyujxe+PehJq/533f3AHDvZe2fJnvf5cOId1r59b924fb4qKrz8dNXdxDrsPKrOaM7rY01q3cMEwYk8c4Xx6is9TZ5zfpDJTisZkb3j++Umlojym5hRL84NueVtfr4m/xSN/uLqnRUikQshc7T+K31B+l63eEtRERERLo31zqI6wcJGc1esvbACV7bcpRLhqdw6cjUFt3WajHz9DfGM6h3DL9csbOx7bLB+kMlfLTnONd+JY2Rae0PX8kxdu69bBgF5bU8u+oAj7+9m6PltTxw5XD6JUS1+/6tcf34dGq9Ad7afuyMz/n8ATYdLuUrGYk4rOcO7+GQk5FEcVUd+aWtG2r55VEpKaEoS6TdFDpPE6gPnYZWOkVERCRUaiug8IvgKmczK1Nef4CHX9+Bw2rmkWvOHB50NgnRNv44bwKxDit3/20T+4uqgOARJo+/vRuL2cQPL83ukC8F4ObzMxneN45FHx/kxc8OMykrmZsnZXbY/VvqqrH9sFvNTU6x3X2skmqPn0kR2FrbICczEYDNrta12H685zhmE1w4pPUDoUQ6g0Ln6WzB0OmpqQpzISIiItJtHdkAGJAxudlLlqzOZV9RFd+/eAiZvaJb/RKD+sTy+5vPw+3x890/r6fM7WFiI2P7AAAgAElEQVTV3uOsP1TK1yZkMLB3TDu+gFNZLWZ+ds0oPP4ANrOZ38wd06ojWDpKQpSNy0amsi63BFfJqV1r6+vP55wQgUOEGuTUDxPadLjlw4Q8vgD/PnCCnMwkEqJtoSpNpF0UOk9XHzrrqs8c/S0iIiLSIc4xRKigvIYnPtjHgF7RzJ8+qM0vc+HQ3jxyzUgOnXDzvZc28T9v78FuNXPPzCFtvmdzLhjci199dTTP3Dy+2Qm7naHhzM5/bjpyyuMbDpViMsH4AZEbOrN6RZMUbWvVSuemvFKq6nw6KkUimkLnaUz152RppVNERERCJu8zsDqh75gmP/3LFTtxe/z8/NpROG3t23/4zQuy+ObkAaw9eIKdBRXMu2BAyPZa3jJ5QIv3nobKRUN60yfOwT835zcO5DEMg/WHShjeN554Z+SuBppMJnIyk9h5tJxar79Fz/lyP6dCp0Quhc7TmOzBVhNvrUKniIiIhEDAD/kbIG08WM88v/KTvcdZuf0YV4zqy8XDOmYwzMPXjGR6dh/6xDn43sUdv8oZSawWM18dl8bhE2421repukpqKKqsi8jzOU+Xk5GI12/wxdGWdd19vOc4yTF2xvRPCHFlIm2n0HkaizMYOn0KnSIiIhIKRbvAU9nkUSl1Pj8/e+MLomwWHr5mZIe9pM1i5oVvT+ST+2aQHHNm0O1uGlpsG87sXNe4nzNyhwg1aNjXuTnv3Ps6iypq2VlQwUVDe4dlD61ISyl0nsbiDLbX+hU6RUREpI32FVby+pYj7C+qxB847czFxvM5z9zP+YePD5JbXM09M4eSltixLbAmk6lF53x2B8P7xjOyXzwrthVQ6/WzoT50doWVzq9kJGAytWyC7Sf7igG11krks4a7gEhjawiddQqdIiIi0nqGYXDHixvJLQ4evxZttzAqLZ4x/RMZkx7PzH1riIczVjpdJW6e/mg/Q1Jiue3CgZ1feDcz97x0frliJ+/tLGT9oRLSk6I6/dzQtohz2shOiWNzCybYNuznvGioQqdENoXO09ii4gAwPO5zXCkiIiJyps8OlpBbXM3lo1LJ6hXD9iPlbD9SzvpDwRCxyr6aElM/7n9xH6P7FzEmPRhIH3trF3W+AL+4bhR2q5rR2uu6cWk8unIXi1fncuB4NXNy+oe7pBbLyUxk6XoXx8pr6ZvgbPIaf8Dg033HGd0/nj5xjk6uUKR1FDpPY48KrnQanuowVyIiIiJd0Svr8wC4/4rhjUeHBAIGeSVu9hw4QNZbhXwcdSm7jlU07jVscO1X0pgyuHen19wd9Y51cHF2Hz7YXQRE9vmcp2sInVtcpVyR0K/Ja7bll1Hm9nLz+ZmdXJ1I6yl0nsYZHVzpNCl0ioiISCuVu72s3HGMSVnJp5xVaTabyOodQ9bxYCCdPvMqto6/jLwSN9uOlLPjSDnHymv56dUjwlV6tzT3vPTG0DmxCwwRavDlMKEyrhjddOj88qiUjplwLBJKCp2ncTqj8RlmTD6114qIiEjrvLo5H48vwNcnZjR9QcMQoczJXwbR3jFc+5W0ziuyB5k5IoWEqOC5nENO+iVApBvSJ5Y4h5XNec0PE/p473HiHFZyMhM7sTKRtlHoPE2M04obB2ZvTbhLERERkS7EMAyWrncR57Qye0zTq1O41oEjAXoP69zieiiH1cJT38jBHzC61JEiZrOJr2QksuFwCV5/AJvl1D2+pdUetrrKuGxk3zM+JxKJ9Lf0NFF2CzU4sPi10ikiIiItty2/nN3HKvnquP5NH03iq4OjmyFjIpj1I1hnmZbdhxnDu14L6vjMRGq9AXYXVJ7xudX7iwkYMH2YptZK16D/450mxm7FbTiwqr1WREREANwl8MQ4WL/4rJctXe8CaL61tmAr+OuaPJ9T5HSN+zpdZx6d0rCfc5rO55QuQqHzNFE2C26c2AJqrxURERFg52tQmgsf/AJqy5u8pLrOxxtbjjC6fzyj+yc0fZ+G/Zynnc8p0pRxGcG9mqfv6zQMg4/3HmdoSiz9EyP/3FERUOg8g9lsotbkxO6vDXcpIiIiEgl2/DP4trYMPnu2yUv+ta2Aao+fr088y/EVrs/BZIb+54WgSOlukmLsDOwdw+a8U1c6dxVUcryyjula5ZQuRKGzCR6TE7uh0CkiItLjVRTAodUw7CpIHgxrnwm2255m6fo8nDYz141rZgqtYQSHCKWOAkdciIuW7iInI5FDJ9yUVHsaH2s8KkX7OaULUehsgsfixGHUBr9BiIiISM+18zXAgLFfg4t/DHUVsPbpUy7ZW1jJprwyrhqTRrzT1vR9yg5DVaH2c0qrNByHcvJq58d7i3DazF3q3FERhc4meM3RmDFAx6aIiIj0bNuXgT0Whl4Go+dCn+Hw2XNQXdx4ySv1A4RumtTMACEIrnKCQqe0SuMwofp9nVV1PjYcKuWCQb1w2pqYkCwSoRQ6m+Cz1m/K9mqCrYiISI9VegiObIBhs8EeDWZLcLXTWw1r/g+AOp+ff27KZ3CfGCYMSGr+XhoiJG0wvG8cTpu5cYLtv/cX4wsY2s8pXY5CZxP8lvrQ6akObyEiIiISPg0DhMbc8OVjI66D1NGw7o9QeYx3vyik1O3lpomZmEym5u/l+hxi+0LigNDWLN2K1WJmbHoiW13l+APGSfs5u965o9KzKXQ2IWCNBsDwVIW5EhEREQmbHf8EZyIMmvHlY2YzzHgQfDXw6UJeWe/CZjExZ3z/5u9TVwmFXwRXOc8WTEWakJOZSFWdj31FlXy89ziZydFk9YoOd1kiraLQ2YSALQYAb61Cp4iISI90fA8UboeR14LVfurnhs2GtByMDX/iwP49XDoyld6xjubvdWQjGAHt55Q2yckItm3/c9MR8ktrmJ7d5+yr6iIRSKGzKbbgb488boVOERGRHqmhtXb03DM/ZzLBjJ9gCni42/ra2c/mBMhr2M+p0CmtN75+gu1f1h4C0H5O6ZIUOptiD4bOuhqFThERkR7HMGDHMohJgayLmrzEN/AStpqG8XXrKi7qdY6fF1yfg8UB/caGoFjp7lLinfRPjKLWG8BmMXHB4F7hLkmk1RQ6m2C217fX1lSEuRIRERHpdMe2wYn9MOqrwYm1Tfh4XzGP1d2AFT/mT3/b/L0CAchfD2k5YD1LC67IWTSc1zkxK5kYhzXM1Yi0nkJnE8zOWAC8WukUERHpeXYsD75tqrW23tL1Lj4zRlGXPgW2vgwnDjR94fHdUFeho1KkXRrO61RrrXRVCp1NsNSHTp8GCYmIiPQshhHcz5mQAelNB8Wiilo+3F3E9Ow+OC59GAw/rHqs6fu5tJ9T2u/6nP7cOiWLr03ICHcpIm2i0NkEqyMYOv11OqdTRESkR8lfD+UuGDUneDxKE/6xMR9/wOCmiZkw4AIYfAls/wcU7T7zYte64FutdEo7JMXYeeTaUSTF2M99sUgEUuhsgi0quKczoNApIiLSs2xfFnzbTGttIGDw9w0uesfamTkiJfjgjJ8CBqx69MwnuD6HpIEQmxKaekVEugCFzibYouKC73gUOkVERHqMgB++eBWSB0O/rzR5yWe5Jzh8ws3c89KxWep/jEo/D7KvgJ2vQ8G2Ly+uLoaSA5A5uROKFxGJXAqdTXBExwNgeNxhrkREREQ6zaHVUF0UXOU0mZq8ZOk6FwBfP31v3YwHg29X/ebLx9RaKyICKHQ2yREd3NNp8ip0ioiI9BjnmFpbWu3h7R3HOH9gMoP6xJ76yX5fgRHXwJ6VcGRj8DENERIRARQ6mxTtdOIxLJi9aq8VERHpEXwe2PUGpIyClOFNXvLaliN4/AFumtTMBNGLHwRM8FH93k7XOnDEQ5+m7yci0lOEPHTec889ZGVlYTKZ2LFjR7PX/epXv2Lw4MEMHjyYhx56KNRlnVWMw4obJxafVjpFRER6hIOroKYUxjS9ymkYBkvXuYhzWrlydL+m75E6MrhKuv99yP0Ujm6C9AlgtoSubhGRLiDkofOGG25g9erVDBgwoNlrPvnkE15++WW2bdvGzp07eeutt3jnnXdCXVqzouwW3Diw+GvCVoOIiIiEjtvjwzCMLx/YUT+1dtT1TV6/Nb+cPYWVzMnpj9N2lhB58Y/BZIZX7wRfrVprRUQAa6hfYNq0aee85pVXXuHWW28lJiZ4VMl3vvMdXn75ZS6//PJQl9ekaJuF44aDKIVOERGRbmfZxnx+9I+t2K1mUuIcpMfCC8VvUBw9kte3+kmNzyclzkFqvJPUeAcJUTaWrssDCJ7NeTa9h8LYm2Dr34Ifa4iQiEjoQ2dL5OXlMX369MaPs7KyWLZsWZPXLly4kIULFzZ+XFVV1eH1WC1makxOEvxqrxUREeluNh4uAWDyoF6cqKojs2QVTqOGP5WPZ/E7e8643m414w8YjE1PYGRa/LlfYPr9sP3vwSNY+k/o6PJFRLqciAidAKaTRpOf0u5ymgULFrBgwYLGj9PT00NST53Jid0oCcm9RUREWmPfvn3MmzeP4uJiEhMTeeGFFxg5cuQp19TW1nLnnXeyceNGDMNg0KBBLFmyhN69exMIBLj//vt5++238fl8TJ06lWeffRa73R6mryi8XCU1RNst/PnbE4M/f/z9eYydJuZ//0dcZyRRWFFHYUUtRZV1FFXUUlhRS4nby3/OHNqyF0geCDN/BlWF4GxBSBUR6eYiInRmZmZy6NChxo8PHz5MZuY52ldCzGOOwhGoDWsNIiIiAPPnz+eOO+7g1ltvZdmyZdx2222sXbv2lGsWLVpEVVUV27Ztw2Qycfvtt/P444/z+OOPs3jxYrZt28amTZuw2Wx897vf5YknnuC+++4L01cUXvmlbtKTooKBs64S9r6DacAUUvoPJKWjXmTqPR11JxGRLi8ijky58cYb+fOf/0x1dTV1dXUsWbKEm266Kaw1ec1OHNQFW2NERETCpKioiE2bNnHLLbcAMHfuXHJzc0/5ZW0Dt9uN1+vF5/NRVVXV2A20detWZs2ahd1ux2QyMXv2bF588cXO/DIihj9gcKSshoyk6OADe94KDvwZ3fQAIRERab+Qh8677rqL9PR08vPzmTVrFkOGDAFg9uzZbNiwAYCLL76Yr33ta4wZM4YRI0Zw2WWXccUVV4S6tLPyWqLq39G+ThERCR+Xy0VaWhpWa7A5yWQykZmZSV5e3inXzZ8/n/j4eFJSUkhNTaW8vJy7774bgIkTJ/L6669TWVmJx+Nh6dKlTYbWhQsXkp6e3vgnFHMTwq2wohav3yAjuT50bl8GJguM/Gp4CxMR6cZCHjqfeeYZ8vPz8fl8HDt2jP379wOwcuVKJkz4cnP9ww8/zMGDBzl48CCPPvpoqMs6J5+l/puRR6FTRETC6+S5B9D07IP3338fk8nEsWPHKCgoIDExkV/84hcAfOtb3+Lyyy9n2rRpXHLJJYwaNQqbzXbGPRYsWEB+fn7jn9jY2NB8QWHkKgl+X09PigJ3CRz4AAZdDDG9w1qXiEh3FhHttZEoYK0Pnd7q8BYiIiI9WkZGRuMvbyEYOF0u1xmzD5577jnmzJmD0+nEbrdz880389FHHwHB0Prwww+zefNmVq9ezfDhw88YRNRTuEqDx6FlJEfDrjch4IPRc8NclYhI96bQ2YyALRg6vbXdr7VIRES6jpSUFHJycnjppZcAWL58OVlZWWRlZZ1y3aBBg3jnnXcwDAPDMFixYgWjR48GgpNty8rKACguLuaxxx7j/vvv79SvI1Lkl5600rljOVjsMPyqMFclItK9KXQ2w6gPnXXVlWGuREREerpFixaxaNEisrOzeeyxx1i8eDFw6nyERx55hPLyckaNGsXo0aMpLi7ml7/8JQDl5eVMnjyZUaNGceGFF3LnnXdyzTXXhO3rCSdXSXClM9NeCYc+hSGXQlRimKsSEeneIuLIlIhkjwGgrqaS7rejRUREupJhw4adcUQKBOcjNEhOTmbZsmVNPj81NZXdu3eHrL6uxFXqJiHKRtyBlWAENLVWRKQTaKWzGab60OmpUXutiIhId5Ff4iYjOQp2LANbNAy7MtwliYh0ewqdzTA5gu21PoVOERGRbsHjC3CsopaxMRXg+jwYOOt/ySwiIqGj0NkMqyP4TchXqz2dIiIi3UFBeQ0BAy42Pg8+MEqttSIinUGhsxkWRxwA/lodmSIiItIdNAwRGuw/EHwg68IwViMi0nModDbDGhUcH+SvU+gUERHpDlz1x6Wk1ORCXJqm1oqIdBKFzmbY60NnwKPQKSIi0h3kl7oxEyCmYj+kDA93OSIiPYZCZzPsUcH2WpNHg4RERES6A1dJDRmmIsz+OkgZGe5yRER6DIXOZjiig6HT8LrDXImIiIh0BFepmwlRhcEP+milU0Sksyh0NsNZHzrNCp0iIiLdgqukhvFRBcEPUkaEtxgRkR5EobMZ0U47tYYNs7cm3KWIiIhIO9V4/BRX1THMfCT4QJ9h4S1IRKQHUehsRozdihsHFr8GCYmIiHR1R8qCnUuZ/sOQkAn1R6OJiEjoKXQ2I8puwY0Tq08rnSIiIl2dq6QGC3561R7W5FoRkU6m0NkMu9VMDQ5sgdpwlyIiIiLt5Cp1k2U6hiXg1X5OEZFOptB5FrUmJ7aAVjpFRES6OleJm6Gmhv2cCp0iIp1JofMsPKYoHAqdIiLSTosXL6akpKTx4xMnTrBkyZIwVtTz5JfWMMzsCn6g9loRkU6l0HkWHnMUDkPttSIi0j5PPfUUycnJjR/36tWLp556KowV9TyuUjdj7QWACXprcq2ISGdS6DwLryUKOz7w+8JdioiIdGGGYZzxmN/vD0MlPZerpIZscz4kZYE9OtzliIj0KAqdZ+GzOIPveHVsioiItF1qaiqvv/5648evvfYaqampYayoZ6mo9eKuqSHNd0RDhEREwsAa7gIimd9a/5tQTzU4E8JbjIiIdFn/93//x7XXXssDDzwABFc+33jjjTBX1XPkl9Qw0FSABb9Cp4hIGCh0nkWgPnT666qxhLkWERHpukaOHMmuXbvYtWtX48dWq74FdxZXqZtsU37wA02uFRHpdGqvPYuALRg6a6srwlyJiIh0ZStXrqS6upqxY8cyduxYKisrefvtt8NdVo/hKnGTrcm1IiJho9B5FiZ7DACemqowVyIiIl3Zgw8+SGJiYuPHiYmJPPjgg2GsqGfJL60h23QEw2SBXkPDXY6ISI+j0Hk29Sudde7KMBciIiLdiclk0vTaThRc6cyH5EFgc4a7HBGRHkeh8yxMjoaVToVOERFpu9jYWDZs2ND48fr164mNjQ1jRT1LYUk5A0yFmNRaKyISFppicBZmR/AHAl+t2mtFRKTt/vu//5urr76acePGAbB9+3aWL18e5qp6BsMwsJftx2IOQMrIcJcjItIjKXSehbV+pdNXp3M6RUSk7aZOncoXX3zBmjVrALjwwgtJTk4Oc1U9Q0m1hwzfYbADfbTSKSISDgqdZ2FxBlc6AxokJCIi7dSrVy+uueYavF4vAB6PB7vdHuaquj9XaQ3DzPXHpeiMThGRsGjxns5FixZRXl4OwF133cWECRP45JNPQlZYJLBFxQEQ8GilU0RE2m7jxo3k5OTgcDiIiopq/COhl1/qZqgpn4DJBsmDw12OiEiP1OLQ+cwzz5CQkMCaNWvYsWMHv/71r/nRj34UytrCzh4VXOk0FDpFRKQd7r77bp5++mlGjRpFcXExDz/8MI8//ni4y+oRXCU1ZJvyqUsYCFatLIuIhEOLQ6fVGuzE/fDDD/nWt77F5Zdfjs/nC1lhkcARHVzpxOMObyEiItKl1dXVMXXqVHw+H0lJSfzsZz/jzTffDHdZPcKx4hMMMBdhUmutiEjYtDh0ms1mli5dyiuvvMLMmTOB4H6U7qwhdJq8WukUEZG2a9i7mZyczPbt2ykpKeHw4cNhrqpnMI7vAcCRNirMlYiI9FwtDp1PP/00S5cu5fbbbycrK4u9e/cyY8aMUNYWdlHRMQQME2afVjpFRKTt5s6dy4kTJ/jxj3/M1KlTycjI4M477wx3WT1CdNk+AK10ioiEUYun106ePJnXXnsNCJ551a9fP5566qmQFRYJoh12arArdIqISLvcd999AFx55ZUcP34ct9tNUlJSmKvq/gIBg97uA2BBk2tFRMKoxSudt912G2VlZXg8HsaNG0dqaiq///3vQ1lb2EXbLbhxYPXVhLsUERHpJhwOhwJnJymqrGMwLrwmOyQNDHc5IiI9VotD58aNG0lMTOSdd94hJyeHY8eOsWjRolDWFnYOq5kaHFj9Cp0iIiJdjavUzVDzESpissCio8lFRMKlxaHTMAwAPvnkE66++mri4+Mxm1v89C7JZDJRSxS2gEKniIhIV1NQWES6qZi6pOxwlyIi0qO1ODX27duXO++8k3/84x/MmjULr9eL3+8PZW0Roc7sxB6oDXcZIiLShf35z39u0WPSsWqP7gTA3HdkmCsREenZWhw6//rXvzJ8+HCWLl1KYmIiR44cYcGCBaGsLSJ4zE4cCp0iItIOTzzxxBmPPfnkk2GopIc5vguAuPQxYS5ERKRna/EGh969ezN//ny2bdvGunXrGDNmDLfeemsIS4sMHnMUTn8NGAaYTOEuR0REupBNmzaxfv16Tpw4wR/+8IfGx8vLy6mt1S80Qy22PHhcSkzG6DBXIiLSs7U4dP773//mhhtuIDU1FcMwOH78OMuWLeOCCy4IZX1h57NEYfUGwO8BqyPc5YiISBdy+PBhVq9eTVVVFZ9++mnj4/Hx8SxZsiSMlfUMvWsOUosDZ2JWuEsREenRWhw6FyxYwD/+8Q+mTp0KBEPoD3/4Qz777LOQFRcJfJbo4DueaoVOERFplTlz5jBnzhzefPNNrrnmmnCX06P4/AEy/XkUOgcwoJsPPhQRiXQt/r9wbW1tY+AEmDJlCjU13X+qa8AaFXxbVx3mSkREpKsqLCykvLwcgP/8z/9k8uTJrF69OsxVdW/HCo/R11RKWezgcJciItLjtTh0RkdH8/777zd+vGrVKmJiYkJSVCQJ2IIrnXU1FWGuREREuqonn3yShIQE1q5dy8aNG3nooYe49957w11Wt1Z6aBsAnuRhYa5ERERa3F775JNPMnfuXBwOByaTibq6Ov7617+GsraIYNiCwbq2uoqoMNciIiJdk9Ua/Hb7wQcfMG/ePK666ioeeuihMFfVvdUd3QGATceliIiEXYtD54QJE9i/fz979uzBMAyGDRvGkCFDyMvLC2V94WcPrnR6airDXIiIiHRVJpOJZcuW8corr/DGG28AUFdXF+aqujdL8W4A4geMDXMlIiLS4tAJYLPZGD36y7HjhmF0eEGRxlS/0llXUxXmSkREpKt68skn+c1vfsO3v/1tBg4cyN69e5k+fXq4y+rWYir2U2U4SU0fEu5SRER6vFaFztOZesC5lRZnMHT6tNIpIiJtNHXqVFasWNH4y9rs7Gx+//vfh7mq7i2lNpdcUwZjnLZwlyIi0uOdM3Tu3Lmz2c/5fL4OLSYSmRyxAPhqtdIpIiJts3v3bm655RaKiorIy8tj48aNLF++nEcffTTcpXVP1cUkBspY75zEmHDXIiIi5w6dV111VbOfczqd53yBffv2MW/ePIqLi0lMTOSFF15g5MhTN/XX1tZy5513snHjRgzDYNCgQSxZsoTevXu34EsILZszGDr9OjJFRETa6Pvf/z6//e1v+eEPfwjA+PHjmTdvnkJniHgKvsAOVMSptVZEJBKc88iU3NzcZv8cPHjwnC8wf/587rjjDvbu3cv999/PbbfddsY1ixYtoqqqim3btrFjxw5SU1N5/PHH2/YVdbCG9lp/nVY6RUSkbSoqKrj44osbPzaZTNhsavsMlfK87QD4eum4FBGRSNDiczrboqioiE2bNnHLLbcAMHfuXHJzczl06NAZ17rdbrxeLz6fj6qqKtLT00NZWovZo+IAMLTSKSIibWS1WvH5fI2zEI4ePYrZHNJvwT2a5+gXANj66bgUEZFIENLveC6Xi7S0tMbzyUwmE5mZmWccszJ//nzi4+NJSUkhNTWV8vJy7r777lCW1mK2+tCJR6FTRETa5vvf/z5z586luLiYX/3qV0yfPp0FCxaEu6xuy3piN+VGNL37ZoW7FBERIcShE86ccNvUMSvvv/8+JpOJY8eOUVBQQGJiIr/4xS+avN/ChQtJT09v/FNVFdq2V2dMfej0ukP6OiIi0n1961vf4t577+XGG2+kpKSE559/nptvvjncZXVPhkF8xX72Gulk9IoJdzUiIkKIQ2dGRgb5+fmNU24Nw8DlcpGZmXnKdc899xxz5szB6XRit9u5+eab+eijj5q854IFC8jPz2/8ExsbG8ovAWf9SqdJoVNERFrpG9/4RuP706ZN43//939ZuHDhKfs7W2Lfvn1MmTKF7OxsJk2a1ORk+draWm699VbGjBnD6NGjufbaaykuLgaC33/vu+8+Ro0axdixY5kxYwb79+9v19cWsaqKiPJXsM9IJy3x3AMPRUQk9EIaOlNSUsjJyeGll14CYPny5WRlZZGVlXXKdYMGDeKdd97BMAwMw2DFihWMHj06lKW1WHR0NH7DhEWhU0REWmn37t0dcp/2DuV74403+OSTT9iyZQvbtm1j5syZPPjggx1SW8QpCgbyAvtAHFZLmIsRERHohPbaRYsWsWjRIrKzs3nsscdYvHgxALNnz2bDhg0APPLII5SXlzNq1ChGjx5NcXExv/zlL0NdWotEO6xU48TsV+gUEZHWOX2LSVt01FC+uro6amtrMQyDioqKiBnY1+GOB4N+VbyOSxERiRTnPKezvYYNG8batWvPeHzlypWN7ycnJ7Ns2bJQl9ImTquF4ziw+mrCXYqIiHQx27dvJyUl5YzHDcPAZDJRVFR0znucbSjfyZ1D8+fPZzIks8oAACAASURBVO3ataSkpGCxWDj//PMbh/Jdc801rFq1ir59+xIXF0f//v35+OOPz3ithQsXsnDhwsaPQz03IRS8x3ZiA/y9h4e7FBERqad57edgNpuoxYktUBvuUkREpIvJzs5m/fr1Z/zZsGED69evb/F92juUb9OmTezevZsjR45w9OhRZs6c2eSU+M6emxAK3oIvOGHEkdinf7hLERGRegqdLVBrcmIPqL1WRERax+FwMGDAgGb/tERHDOV74YUXmDFjBomJiZjNZubNm9fswL4uzTCwlexhn5FORnJ0uKsREZF6Cp0tUGeOwq6VThERaaWmViRbqyOG8g0aNIgPPvgAr9cLwJtvvhkxA/s6VMVRbN4q9gbSSU9S6BQRiRQKnS3gNTtxGgqdIiLSOps3b+6Q+7R3KN9dd91FZmYmY8aMYezYsXz00Uc888wzHVJbRCnaBRA8ozM5KszFiIhIg5APEuoOvJYonL46MAzogEmEIiIirdHeoXwOh4Pnn38+ZPVFjOPB0HmADPolKHSKiEQKrXS2gM8ShRkDfFrtFBERiVhFweNSKuOHYDHrl8QiIpFCobMF/NbgvhCjruuNjhcREekpjKKdHDcSiU9ODXcpIiJyEoXOFgjUh05PjUKniIhIRAoE4Phu9gT6k6EhQiIiEUWhswUMW/CbV627MsyViIiISJPKXZi87vrjUrSfU0Qkkih0tkBj6KxW6BQREYlI9ZNr9xgZOqNTRCTCKHS2hD0GAE+NQqeIiEhEqp9cqzM6RUQij0JnC5gVOkVERCJb/eTa/UZ/MpLUXisiEkkUOlvA7AyGTl+tBgmJiIhEpKKdnLD0oc4aR584R7irERGRkyh0toDFHguAr7Y6zJWIiIjIGQJ+KN7LAdJJT4rCZNIZnSIikcQa7gK6AltUMHQGdE6niIhI5Ck9BL5adgTSNERIRCQCaaWzBayNoVMrnSIiIhGnfnLtLn9/0rWfU0Qk4ih0toC9PnQaHoVOERGRiHPS5NoMTa4VEYk4Cp0tYI+KC77jcYe3EBERETlT/eTafUa62mtFRCKQQmcLRMXEA2DyaqVTREQk4hzfTYWjH26cWukUEYlACp0t4IwOrnSavVrpFBERiSiGASW5FFr7A5CRrD2dIiKRRqGzBaKjo/AYFsx+hU4REZGIUlMK3mpcRm9iHVYSomzhrkhERE6j0NkCUTYLNTiw+mrCXYqIiEi3t6+wklqvv2UXl+UBcMCbrDM6RUQilEJnC1jMJtw4sfoVOkVEREJpV0EFl/7uEy757Sr+uSmfQMA4+xPqQ+fumkQNERIRiVAKnS1Ua3JiCyh0ioiIhNK+oioAjlXUsuDvW7nm6dWs2V/c/BPKXQDk+XtriJCISIRS6Gwhj8mJPVAb7jJERES6tcLy4PfaP86bwB3TBrGvsIqb//g5t/5pHXuOVZ75hLJg6Dxi9CY9SUOEREQikTXcBXQVHrMTR+BEuMsQERHp1grqQ2d2ahyXDE/lm5MH8Nt39/D6lqN8svc4N56XwYLLskmNdwafUJZHwGSlkCS114qIRCitdLaQxxJNlKH2WhERkVAqrKjFZIKUuGCozEiO5ombcnjj7qlMGpjMKxtcXPw/q1j47h6q6nxQnkelPQU/Fh2XIiISoRQ6W8hnicKJBwKBcJciIiLSbRWU19ArxoHdeuqPKGPTE3n59sksnjeB/klRPPnhfi7+n1X8//buPDyO+s7z+Kf61i1fkg9J1pixwfiSwYDthTEEQhKuIXES8nBFGRLYECaz4Rl4dnayGy9JGEIymplc4GQhXCEPiQ2TCSFAGK4wGGxOY0zAgI0lWy35UsuS+u7f/lHdbclnS1ap1d3v1/PUU9XVpda3XEjFR7+jons+UrdriiSpgTGdADAuETpzlHCn/3oa51mdAAA4JRiKaFpN4LDvWZalc+fW6/G/O0u3fnqBKsyA/PFevdlXownlXlX6GTUEAOMRoTNHKY/919NYuC/PlQAAUJySKaPu/VFNPULozPC4Xbr8jCb9obVZktRlTdGixtoxqBAAMBL8STBHKa8dOiP9Iflqp+a5GgAAis+evqgSKaOp1UcPnRnlAzskSV++eIX+ZtGpTpYGADgOtHTmyGRC5wAtnQAAOCEzc+2xWjqz0o9L8U9qVpnP7VRZAIDjROjMlbdCkhQLH+YZYQAA4LgFe+3QeaQxnYcIbbfXtY0OVQQAGA2EzhxZvkzopKUTAAAnBDMtnTl2r1XPdkmWVN3gXFEAgONG6MyRy58OnQO0dAIA4IRMS+ewutdWTZM8PgerAgAcL0JnjjKhMxmlpRMAACcEhzumM9RO11oAKACEzhx5ApWSpESkP8+VAABQnDpDYVUHPCr35TC5fmxA6t8l1TY5XxgA4LgQOnPkLbNDZ4qWTgAAHNHVG9W0mrLcDg512OsaWjoBYLwjdObIG6iSJJkYLZ0AAIw2Y4w6Q2HV5zyek5lrAaBQEDpz5C8ndAIA4JRQOK5IPKVpuc5cm31cCt1rAWC8I3TmyFdud6+14gN5rgQAgOIzoplrJamG0AkA4x2hM0dl5dWSCJ0AADihc7gz19K9FgAKBqEzR4EKu3utm9AJAMCoG9HjUiqmSN4cJx4CAOQNoTNH5QGfIsYrV5LQCQDAaMuGzlzHdPa0M3MtABQIQmeOvG6XwgrIkwjnuxQAAIpOJnROy6WlMxGT9ncyiRAAFAhC5zCELb+8KUInAACjLdgbUcDrUk2Z99gH93ZIMoznBIACQegchqhVJh+hEwCAURcMRTS1OiDLso59MDPXAkBBIXQOQ9QKyJ+K5LsMAACKTmcoPIKZawmdAFAICJ3DEHcH5DeETgAARtNALKHeSELTanKciTaUbumkey0AFARC5zDEXWUKKJrvMgAAKCqZSYTqc565Nt3Syey1AFAQCJ3DkHCXy6+4lEzkuxQAAIrGsGaulewxnYFaKVDtYFUAgNFC6ByGhMfu9pOI7M9zJQAAFI9g7zBbOkPb6VoLAAWE0DkMxlMuSQoPEDoBABgtncNp6UwmpNAOqXamw1UBAEaL46Fzy5YtWr58uebMmaPTTz9dmzdvPuxxzz33nE477TTNmzdPJ510ktatW+d0acOW8tqhM9Lfl+dKAAAoHsPqXru/UzJJxnMCQAHxOP0NrrvuOl177bVqbW3VmjVrdM011xwSKHfu3KkvfvGL+sMf/qC5c+cqEokoEhmHs8T6KiRJ0YHePBcCAEDxCPZG5HZZmlTpP/bBzFwLAAXH0ZbO7u5uvfbaa7ryyislSStXrtTWrVu1bdu2Icf99Kc/1ZVXXqm5c+dKkgKBgGpra50sbWTSoTMWpnstAACjJRiKqL7KL7fLOvbBPKMTAAqOo6Gzvb1d06dPl8djN6halqWmpiZt3759yHGbN29WOBzWeeedp5aWFv3t3/6tBgYGnCxtRFw+u3ttbIDutQAAjJZgb0RThzNzrUT3WgAoII6P6bSsoX+1NMYcckw8Htezzz6r3/zmN3rllVcUCoW0atWqw35eW1ubGhoasktf39gFQJe/0q43SugEAIydXOZHiEQiam1t1YIFCzR//nxdcskl2r17tyTpvvvuU0tLS3aZPHmyPvOZz4z1aRxWLJHS7r5o7qEzREsnABQaR0NnY2OjOjo6lEjYz7U0xqi9vV1NTUNvFDNnztSFF16oCRMmyOPx6Atf+ILWr19/2M+88cYb1dHRkV0qKyudPIUh3AH7eyXDhE4AwNjJzI/w3nvv6eabb9Y111xzyDGrV69WX1+fNm7cqE2bNqm+vl633367JOnqq6/WG2+8kV2mTZumK664YqxP47C690dkjDS1uiy3L+jZLvkqpbIJzhYGABg1jobOuro6LV68WA888IAkae3atWpublZzc/OQ4y6//HI988wzikajkqTHH39cixYtcrK0EfEE7DGdSVo6AQBjJNf5ESRpYGBA8XhciURCfX19amhoOOSY9evXq6urS5dcconTpeekq3cYM9dKdvfamkbJymH8JwBgXHC8e+3q1au1evVqzZkzR7fddpvuuusuSdIFF1ygV155RZK0fPlyXXzxxWppadGCBQu0a9cu3XLLLU6XNmzeQJUkycTG33hTAEBxynV+hOuuu07V1dWqq6tTfX29QqGQbrjhhkM+76677tJVV10lr9c7JvUfS+YZnfW5hM5Uyp69lq61AFBQHH9kyoknnnjYZ24+9thjQ17ffPPNuvnmm50u57j4yjKhsz/PlQAASkku8yM89dRTsixLwWBQLpdLra2tuuWWW4bMkTAwMKCHHnpIL7744mG/T1tbm9ra2rKvx2LehGE9o7O/W0rGeFwKABQYx1s6i4m/Ij1+lNAJABgjuc6PcOedd+rTn/60AoGAfD6frrjiCj3zzDNDjlmzZo3mzp2rk08++bDfKx/zJmRC59TqHEInM9cCQEEidA6Dv9xu6bTidK8FAIyNXOdHmDVrlp544gkZY2SM0aOPPqr58+cPOebuu+8+7CRE+dSZHtNZV+0/9sE9H9lrutcCQEEhdA5DWTp0uhKETgDA2MllfoRVq1YpFApp3rx5mj9/vnbv3q1vf/vb2c/44IMP9Oqrr+qyyy7LyzkcSTAU0eRKn/we97EPDqVbOgmdAFBQHB/TWUzKKiqVMhahEwAwpnKZH2HixIlas2bNET/jhBNO0P79+x2p73gEQxHV59K1VqJ7LQAUKFo6h8Hn8SgsnzyJcL5LAQCg4KVSRl29kWE8LmW75AlIlXXOFgYAGFWEzmGwLEsRKyBPkpZOAACO157+mBIpo6m5hs5Qu1TTwDM6AaDAEDqHKaKAvKlIvssAAKDgDWvmWmPslk7GcwJAwSF0DlPEFZCP0AkAwHHrDNnDVabWlB374IG9UnyA8ZwAUIAIncMUdwUUSDGmEwCA49XVO4yWztB2e11L6ASAQkPoHKaYq0wB0dIJAMDx6sx0r81lTGdPJnTOdLAiAIATCJ3DlHCXKWCi+S4DAICCFxxW6ORxKQBQqAidw5T0lMtrJZWMEzwBADgewd6IqvweVfpzeGx4KB066V4LAAWH0DlMSU+5JCnc35vnSgAAKGzBUCT3x6X0bJdcHqlqmrNFAQBGHaFzmIzHnmEvQugEAGDEjDHqHFbobJeqZ0gut7OFAQBGHaFzmFLeCklSdGB/nisBAKBw9UYSCseTuc1cK9mz1/KMTgAoSITOYbJ8duiMEDoBABixzCRC03Jp6YyE7IXQCQAFidA5XD57TGc8TOgEAGCkgulndNYzcy0AFD1C5zC5/ZWSpHi4P8+VAABQuIKhsKQcWzqzz+ikpRMAChGhc5hcfrt7bSLal+dKAAAoXJ2ZZ3RWlx37YB6XAgAFjdA5TJ6A3dKZjBA6AQAYqa5099qcZq/NtHTSvRYAChKhc5gyoTMVpXstAAAj1RmKyOdxaUK599gH92yXLJf9yBQAQMEhdA6Tr4zQCQDA8QqGIppWE5BlWcc+ONQuVU2TPD7nCwMAjDpC5zD5yqrsjRihEwCAkQr2RlSf6zM6e9rpWgsABYzQOUz+inTojA/ktxAAAApUOJZUz0A8t5lrY/3SwG5mrgWAAkboHKZAuR06XXFaOgEAGIngcCYRCnXYa2auBYCCRegcpvLKakmSKxHOcyUAABSmYPZxKbnMXJt5XAotnQBQqAidw+T3lylpLLmTdK8FAGAkgr32H25z6l7b85G9ZkwnABQsQucwWS6XBhSQh5ZOAABGJBiKSlJuEwmFaOkEgEJH6ByBiBWQN0XoBABgJIKhTEtn2bEP7tlur2saHKwIAOAkQucIRK2AfElCJwAAI9EZisjtsjSlyn/sg3vapYo6yZtDQAUAjEuEzhGIugLymUi+ywAAoCB19UY0pdIvt8s69sGhdmauBYACR+gcgZirTAFCJwAAI9IZiuT2uJREVNrfyXhOAChwhM4RSLjLVGYiSqVMvksBAKCgxJMp7eqL5jZzbeYZncxcCwAFjdA5AtGqJlVYET31i2/JGIInAAC52rU/KmOYuRYASgmhcwTmXfnP6nA36rztP9STa36e73IAACgYnSF7eEpuz+hMz1xL6ASAgkboHIGymkmq/sp/qMdVqxWb/peeeuI/8l0SAAAFIZgOnTmN6exJt3TSvRYAChqhc4Sqp86SLn9IKcutU178qv700kv5LgkAgHEv2JsOnbl0r822dBI6AaCQETqPw8TZZ2j/RT9XjTWgpj9crQ1vv5fvkgAAGNeCIfs519NqcnjuZqhdKpsg+ascrgoA4CRC53GqX3KJgmd+VzOtLvl+fbne/iiY75IAABi3gr1RSVJdtf/YB/e007UWAIoAoXMUzDjveu2Y/1Utsrao+56rtK27N98lAQAwLgVDYU2s8CngdR/9wGRC6t3BJEIAUAQInaNkxmdu1c6mi3SOWa9Xfvbf1Z3uPgQAAA7oDEVyG8+5f6dkkoROACgChM7R4nJp+tV3q3viEn028Xs9csc/KjQQz3dVAACMG6mUUXdvdHgz1xI6AaDgETpHk8evuq+s0d7yv9BXwnfr56vbFI4l810VAADjwt6BmGLJVI6hMz1zLWM6AaDgETpHW9kETfjKb9Xvm6i/7fm+2u6+X/FkKt9VAQCQd9lndObSvTaUaekkdAJAoSN0OsCaMFMVrWsll0df7fxH/eDB3yuVMvkuCwCAvMqGzuG0dNK9FgAKHqHTIa4Zi+X+wr2qsQZ0+ZYb9a+//S8ZQ/AEAJSuzl47dE7LNXT6qqRArcNVAQCcRuh0kOfETyj5qR9opqtbH3v967rtt68pSYsnAKBEBdMzu+fcvba2UbIsh6sCADiN0Okw3xnXKLLsG2pxfaAzX/26brj3RQ3EEvkuCwCAMRcMRSXl0L02lZJCHXStBYAiQegcA4Hzv6XkKV/SWe5NuuzDf9AVdz6nrnQXIwAASkWwN6xKv0dVAe/RD+zrkpIxZq4FgCJB6BwLliX3RW0yi6/W2e43dcOub+uzP35W73T25rsyAADGTDAUUX21/9gHMnMtABQVQudYcblkXfxv0qLLda77df3vyPf1hTue17Pvdue7MgAAHGeMUWcoomk1Zcc+mJlrAaCoEDrHkssl/fWPpQWf0/muV/R914917b0v6/6XPsp3ZQAAOGp/NKGBWHJ4j0upIXQCQDHw5LuAkuNyS5feKaUSOv/tR/TTMo+u/ffr9NHufv3DBXPldjFLHwCg+HRlntGZ68y1Ei2dAFAkHG/p3LJli5YvX645c+bo9NNP1+bNm4947K5du1RfX6/PfvazTpeVX26P9JmfS3Mv1nnJP+kXtb/Q3S98oP/+wKvMbAsAKEqdmdCZa0unp0yqmOxwVQCAseB46Lzuuut07bXX6r333tPNN9+sa6655ojHXn/99brgggucLml8cHullXdLcz6lFZH/1IP1D+qpzZ26bPVL6mZmWwBAkQkOp6Wzp12qaeAZnQBQJBwNnd3d3Xrttdd05ZVXSpJWrlyprVu3atu2bYcc+8tf/lL19fVasWKFkyWNLx6f9Pl7pb/8uJaGHtO/z3xYb+3o0aU/+S/9OcjMtgCA4hHszbGl0xi7ey1dawGgaDgaOtvb2zV9+nR5PPbQUcuy1NTUpO3btw85bufOnWpra9Ntt912zM9sa2tTQ0NDdunr63Ok9jHj8UuXPSDNOkeLuh7Wf570qHb1RfTZO9bp7he2atOOkOLJVL6rBADguGS61047Vugc2CPFB3hcCgAUEce711oHdY0xxhxyzFe+8hXdfvvtqqysPObn3Xjjjero6MguuXzNuOcNSF94UGo+Syds+5WeX/hHuS3plkc366IfvaAFq57Q5+58Ubc+9o4ee6tTnaFwvisGAIyhXOZHiEQiam1t1YIFCzR//nxdcskl2r17d/b9t956S2effbbmzp2rE088UQ8//PBYnoKCobB8bpcmVviOfmD3O/a6htAJAMXC0dlrGxsb1dHRoUQiIY/HI2OM2tvb1dQ0tMvMunXrsmM9+/r6FA6H9YlPfEJPPPGEk+WNL75y6fKHpAc+q2nv/ELrz6jUCzNv0BsdIb2+vUdvtvdow7Z92cPrq/1a3DhBLU21amms1cKGGpX7mIwYAIpRZn6E1tZWrVmzRtdcc43WrVs35JjVq1err69PGzdulGVZ2T/o3n777RoYGNCll16qe++9V2eeeaYSiYT27dt3hO/mjGBvVPU1/kP+GH2IF9okWdJJF45JXQAA5zmaUurq6rR48WI98MADam1t1dq1a9Xc3Kzm5uYhx+3duze7fc899+jRRx/VmjVrnCxtfPJVSFf8Wnpgpfwv/0jn7n1P557wMenCZUpNOVcf7Anr9fYevdHeo9e39+jJzUE9/nZQkuR2WZpdV6k59VWaU59ZV6lxYjmPYQGAApaZH+HJJ5+UZM+PcMMNN2jbtm2H3E8HBgYUj8flcrnU19enBQsWSJIefPBBLVu2TGeeeaYkyePxaMqUKWN6HsFQWLPrqo5+0NY/SR88LS28TKqbOzaFAQAc53jT2OrVq9Xa2qpbb71V1dXVuvfeeyVJF1xwgW655RYtWbLE6RIKi79KumKNtOZL0pYnpS12a6/LX63ZjadrdtMyff6U5dJFp6k/5dFbO0LpELpPm3b06j/e3Dn04zwu/WVdpU6sr9LsQYF0Rm2ZXIRRABj3jjY/wuDQed1112ndunWqq6uT2+3WGWecoRtuuEGStHnzZgUCAV100UXq6OjQwoUL9c///M9jFjwj8aT2DcRVf7TxnMZI/3mL5PJIZ//DmNQFABgbjofOE0888ZAuQJL02GOPHfb41tZWtba2OlzVOBeolq5cK0VCUvt66aMXpe3rpK3PS+8/ZR/j9qtixila2rRMS2cul844XQrUqC+a0Jau/drS1af3uvbr3fT2wzt3DPkW5T63ZtdX6VPzp+rzSxqPPcYGAJA3ucyP8NRTT8myLAWDQblcLrW2tuqWW27RqlWrFI/H9cQTT+ill17S9OnT9c1vflNf+9rX9Otf/3rIZ7S1tamtrS37erQm6+vqzWESofeekDrWS6d9WZr4F6PyfQEA4wODAMezQI00++P2IknxiLTztQMhdPvL9vqFNslySXXzVNlwqhbPWKLFM5dIp54kuey5okLhuLZ07dd76TC6pXu/Nu3o1W1/+LPa/vieLlowTVcum6nFjbXHHm8DABgzuc6PcOedd+rqq69WIGAHuyuuuEK33367Vq1apZkzZ+qcc87RjBkzsu8d7rnYN954o2688cbs64aGhlE5h8zMtfVHekZnKiU9/W3JUyb91U2j8j0BAOMHobOQeAPSzOX2IkmppNS1SfponbT9RTuEvnqPvUiSr0qasViasUQ1DUu0ZMYSLWk+8D8psURKj78d1APrPtLDr+/Qw6/v0MnTqnXVspn665bpTEwEAONArvMjzJo1S0888YQ+97nPSZIeffRRzZ8/X5L0+c9/XnfddZd6e3tVXV2txx9/XIsWLRqzczhmS+emtfb97L/9D6lq6pjVBQAYG5Y5XB+dAtLQ0KCOjo58lzE+GCP17pB2vCp1vGKvd75uP+8so6ZRmnGq1LBEmrHEXru9+nOwVw+89JEeeW2H+mNJVQU8WnlKg65cOlN/WVcEj6UBULKK4T7x7rvvqrW1VXv27MnOjzBv3rwh8yPs3btX1157rTZv3izLsnTyySdr9erVmjhxoiTpvvvu0/e+9z15PB7NmDFDP/vZz47Zkjla/3Z3PveBbvvDn/Xw9ct1StOEoW8m49KPT5MG9kp/94ZUPvG4vx8AYOzkcq8gdBa7ZELa9U46hL4idbwq7fqzpPRlL5sonfzX0vyV0szl6osbPfL6Dt2/bpve67LH8iw/YZKuWjpT551cL6/b8Ue7AsCo4j4xcqP1b7fqP97WPS9u04v/82OaXls29M1X7pYe/Yb0sW/StRYAClAu9wr6TxY7t0eausBelnzJ3hfptVtAt6+TNv9WevUX9lJZr8p5n9ZV81fqyr87Sxs+6tH9L32kxzd16sUP9qi+2q+zZk9Rpd+jCr9b5T6PKv0elfvcqvB77MU3aL/frUq/R36Pi3GiAFDCgqGIXJY0pco/9I14WHrudqliinTGV/NTHADAcYTOUhSolmatsJez/6fUtVl6+2F7TM3Ld0ov3ymrpkmnz7tUp69Yqe4Lz9GvX+nQgy9v15pXh/8Xb5/bpeoyr2rKPOm1vVQHBm2XedJrr/5ySqXqjjTZBACg4HT2RjSlyn9ob5n1P5f2d0qf/J7kZygHABQrQiek+pPt5Zx/lDrftMPn249IL/5QevGHqpt4gm6Y/xl97W8+o76a2RqIJdUfTag/mlR/LKGBWEJ90aQGogn1RRP2+7GE+qMJ9UUS6o0k1BuOKxSOa8e+sN7p7FUknjpqSXVVfi1sqNH8GTXZdV0VQRQAClEwFNbUg/+YGOm1Z1+vaTzQEwcAUJQInTjAsqTpLfZy3v+VOjbYLaBvPyI9/31Zz39fVZX1qqpplGpm2P+jUNMgVc+QJjfYrysm259zDNFEUqFwXL3hRHodV28krr39Mb3XtV8bO0J69t1deuqd7uzXTK0OZEPoghk1WtBQo8mV/qN8FwBAviWSKe3aH1VLY+3QN9b9WArvk87/juThdzkAFDNCJw7P5ZKazrCXT9wqffRf0qaHpeBGKdRhT0p0OG6/HUirM6E0vV09Q6qebi9lE+T3uFVX5VZd1ZFLiMSTeje4Xxt3hLSpI6SNO0J65t1uPfVOV/aYaTUBzZpSoSq/VxV+jyr9blUGPOltT3r86aHbZT63yrxu+TxMjAQATtrVF1XKaGhLZ/9uad1PpMlzpIVfyF9xAIAxQejEsbnc0l/8lb1kJGLS/p12AB289O6w151vStv+dPjP85QdCKDVM9LBdPqBYDrxBMlfqYDXrUWNtVo0VMwvyAAAElZJREFU6K/jkXhS73T2atOOkDZ2hPTWjpA2tofUF0toJPMwe1xWNoCW+9wKpNflPk92u8zrto/xuVU+aDvzNWU+zyFfH/C65XFb8rld8rgsuV0WkykBKEnBkP2Mzqk1g2at/VObFOuzh3W4+V8RACh2/KbHyHh80oRmezmSSEgK7bDDaW9m2XFgO7jxyMG0dqZUlx5rWneyVDdXmjRbAa9Pi5smaPFBz3kzxmTHmu6PpseTpseU9sfsdV80md0fjiU1EE8qHEsqHE+/jiUViSe1tz+mcPq9RGp0nihkWZLX5ZLXbcnjdsnrtre9blc2nPq9bgU8LgW8bgW8LpV53eltd3ZfwOtO77c/w7Ikl2UHWld622Up/frAPsuyJ3SaWOnTpAq/JpR75eHxNwDGQCZ0TqtJt3SGOqQN/0+atkiae0keKwMAjBVCJ5wTqLGX+pOPfEy0z565MBNGQx3Srnel7s3S+3+U3vvDgWNdHmnS7HQQnZsOoydLtTNluVzZx7bUjeIpxJOpbBgdiA0OqSkNxBLZcBqOJ4ccF4knlUgaxZMpxVNG8URKiVRKsaRRIpmy96ffTySNoomUesNxRRIphWNJRRLJEbXc5sqypNoyryZV+jWpwqfJlX5NSgfSSZU+Ta70aWKFP92S6x4SdgMet1wuWm0B5KYzHTrrM91rn/uelIxK5/4feygHAKDoETqRX/5KyT9bmjz70PcSMWnP+3YA7d4sdb8jdb1tz647hCX5KiR/leSrtD/TVyn5qwdtV6bfr7LXmUBcVpverrX3H9QF1ut2qabMpZoyr3P/BodhjFEsmVIkllIkYYfYcDypSPxAKE0kjVLGyBgjY6SUkVImsy+zrewx0URKe/pi2tsf057+qHb3xbSnL6p3u/br5a17h1Wfz+NSwOPKBtKAx61Auvtx5hmuFX63KnwelWee35pe289ztZ/jGvC4lTJGyZRRImXXnkhmXqey+7Pvp4wsS3K7LLktu9uyx23J7XINee2yrGy35gq/R9NqAgp43Q5dLQBH09U7qKVz9/vS67+UZv436YRz81wZAGCsEDoxfnl8Bx7nMli0T9r1ZzuIdm2WejvsfbE+e923S4p9aG+bZO7fz3IdCKAHB9LyiVL5ZHt23vJJ9lIx2d7nKx/d85bdPdbvccvvcatGzgfeeDKlff0x7emPaU/fgVAajiXsoBs/EHyj8dSgEGwH4Ug8qf29cQ3EkxqIJhVLHv2ROPkwscKnqdUBTa8NaFpNmabVBjS9pkzTagKaXlum+uoAE0sBDogmUvK5XZpaE5D+/bv27+Vzv5XTTOcAgOJgGeNkJz7nNTQ0qKOjI99lYDwyRkpE0oF0vxTdb29H99vjTSMhKdJjr8M9B7YjPVI4/X40dOzv4y1PB9JJ9rp8klQ2QfIGJM+g5VivPX57cfsPbLs8Bfk/ZrF0N+HM81r7Y/ZzXPtjyfRzXRMaiNrB1Z1ukfQMWbvkdklul2vIfpfLkjEa0hKaTB2ppdR+3RuJq7Mnos5QWDt7IurqjRx2rK5lSZMr/ZpaHchOBhXwuuT3HBhPG0iPu/V73fJ7DuzzDxqLO/j4zP7BayaUGnvcJ0ZutP7tUikjV9dGafVfSXM+KV3+0ChUBwAYD3K5V9DSieJlWZK3zF40ZWSfkUqmQ+k+e4r/gd2D1nsOer1b6v6zlAiP5kkcGkbdvgOh1Vtudy32ltstrt7yw+yrSK/L7BDr8tprt+for13u9HrQtuXKKQT7PC75PC7VlI9tt+RcJFNGu/ui2tkTVmcoop09YQVDEXs7FFZ3b1Qd++wWXifG1vo89iRRmVmR7fGyruyMyfbMyOmJpHxulXs9KvO5VBXwqqbMq9oyr6rL7O2acq8qfR7G2GLcc7ks6env2C8+9s38FgMAGHOETuBoXO5019qJ0qQTcvuaWL/dcpqIHFjikaGvE1EpHrbXibD9fjJqj2NNRtP7owdtx4buj/TY3ys+YL83VixXOoBmQumg125vOhT709uDg/Jh9rl9g4KsZa8ty96nI2xnF8v+nkP2pReX+6Dj3dn9bsutestSvcutxW63NNkl1bkHHeOS5JUsS0ZSImUplrQngYonjaIJo3jKKJqUYgmjaHZ/KruOJVPp9+yviyUOvI6m1+GEUSRuFIkYhftSCsWNOuNG8ZSUkiVjLKXkkpGy6/QFkCSZ9NqypMqAV1VlXlUHvKpKb/vdbvufIH2QJUvWoHBquSxZ6c+xZzd2a2pNQNNryzVjQrlmTChTTZnvwL99+vsNuVY6KOwe9AcJI6lnIK7OkN3CbGTUPKlSjRPL5c3MnkzLb2n4aJ205Ulp/melqQvyXQ0AYIwROoHR5quwl7GUTEjxfik2YIfQ+EB6+6B9qYR9bCqzxO3W3GR80L5E+nVcSqXs8VeppL0/u51MbyeGvk7GD4Tn6H4puXdoYE5Gx/bf5ThZkrzpZUyu6Egbho2kgfQyjliSJqSXo8xhfXSX/1qa84lRqwl5YIz0n//X/sPOOf8r39UAAPKA0AkUA7dHcqdn5B3PjLGDaiaIJmP2PpOSlF4PeZ1esu8Net8kD7MvlQ7AqUFLMv19kwdeH3xMJjRntjX4+x60fch7mUmTMvvT25nzPdL2Ied10HkM/vzM9zjs5x/6OmlSSqXHrWaG7WeH72deH/w1SfsxQP3RuAaiSQ3E4hqIJuxHA8USMpIsmfRibw82+LXX7VLZQc+VDXjdkjHqTz9Ptz/92alB/ZctSS5L2dmNK3welccq1XSE/5xQIN5/Stq+Tjq1NfceIwCAokLoBDB2LCvdxXb8jfUsJu70MlwVOvzo50QypWBvRB37wurYF1b73gHt7Amrwu/R1JqAptUEVFdtr+urc388TTJltLMnrG17+rV199ClY19YyZTRfb45hM5C9/r9drf6v7o535UAAPKE0AkAOCqP26WGCeVqmDC6jwdyuyw1TixX48RynTV7aNyNJVJq3zegqdWBUf2eyIOVd0udb0o1M/JdCQAgTwidAIBxx+dx6YQplfkuA6PB7ZEaTs13FQCAPOJJ6AAAAAAAxxA6AQAAAACOIXQCAAAAABxD6AQAAAAAOIbQCQAAAABwDKETAAAAAOAYQicAAAAAwDGETgAAAACAYwidAAAAAADHEDoBAAAAAI4hdAIAAAAAHEPoBAAAAAA4htAJAAAAAHAMoRMAAAAA4BjLGGPyXcTx8Pv9mjJlynF/Tl9fnyorK0ehosJSiuddiucscd6lhvM+YNeuXYpGo3mqqLBxjz0+nHdp4bxLRymes3Tk887lPlvwoXO0NDQ0qKOjI99ljLlSPO9SPGeJ8y41nDfGk1K9Lpx3aeG8S0cpnrN0fOdN91oAAAAAgGMInQAAAAAAx7hXrVq1Kt9FjBfLli3Ldwl5UYrnXYrnLHHepYbzxnhSqteF8y4tnHfpKMVzlkZ+3ozpBAAAAAA4hu61AAAAAADHEDoBAAAAAI4p+dC5ZcsWLV++XHPmzNHpp5+uzZs357ukMdHc3KyTTjpJLS0tamlp0UMPPZTvkhzx9a9/Xc3NzbIsS5s2bcruL+brfqRzLvZrHolEdOmll2rOnDlqaWnRJz/5SW3btk2S1N3drU9+8pOaPXu25s+frxdeeCG/xY6io5332WefrVmzZmWv+b/8y7/kt9hRdv7552vhwoVqaWnRWWedpTfeeENScf98F5pSvRbF/vs2oxTvsVJp3me5x3KPPe57rClx55xzjvnFL35hjDHmN7/5jVm6dGl+CxojM2fONG+99Va+y3Dcc889Z9rb2w8532K+7kc652K/5uFw2Pz+9783qVTKGGPMj370I/Pxj3/cGGPMl770JfOtb33LGGPM+vXrTVNTk4nH4/kqdVQd7bxXrFhhfve73+WzPEft27cvu/3II4+YxYsXG2OK++e70JTqtSj237cZpXiPNaY077PcY7nHHu89tqRDZ1dXl6mpqcn+YKRSKVNfX2+2bt2a38LGQDH/YjycwedbKte9lG6Gh7NhwwZzwgknGGOMqaioMN3d3dn3TjvtNPPMM8/kqTJnDT7vYr8hDnbPPfeYU089tWR+vgtBKV+LUvt9W4r3WGNK+z7LPZZ7rDHD+/ku6e617e3tmj59ujwejyTJsiw1NTVp+/btea5sbFxxxRVasGCBvvzlL2vXrl35LmfMlPJ1L6Vr/sMf/lAXX3yx9uzZo1QqpSlTpmTfa25uLtrrnTnvjJtuukkLFizQZZddpg8//DCPlTnj6quvVmNjo775zW/q3nvvLemf7/Gm1K9FKf2+HYzrXhrXnXusjXts7j/fJR06JfsfazBTIk+Qef755/Xmm2/qtdde06RJk/TFL34x3yWNqVK87qV0zW+99VZt2bJF3/3udyWVzvU++Lzvv/9+vfPOO9q4caPOOussXXTRRXmucPTdd999am9v13e+8x3ddNNNkkrneheCUr0WpfT79nC47sV93bnHco8dLOfr7VArbEHo6uoy1dXVJdEF5Gh27txpKisr812Gow7u+lMK1/1o3XyK+Zp///vfN6eeeuqQsQjl5eVF3/XncOd9ML/fb3bv3j2GVY2tQCBggsFgSfx8F4JS+V17LMX8+zajFO+xxpTmfZZ7LPfYkf58l3RLZ11dnRYvXqwHHnhAkrR27Vo1Nzerubk5v4U5rL+/Xz09PdnXv/rVr7R48eI8VjS2SvG6l8o1b2tr069+9Sv98Y9/VG1tbXb/5z73Of3kJz+RJG3YsEHBYFBnnnlmvsocdYc770Qioa6uruwxa9euVX19vSZNmpSvMkdVb2+vdu7cmX39yCOPaNKkSSX58z1eleq1KJXft0fCdbcV43XnHss99nh+vi1jirQNPEfvvvuuWltbtWfPHlVXV+vee+/VvHnz8l2Woz788EOtXLlSyWRSxhjNmjVL//Zv/1aUN4Svfe1r+u1vf6tgMKjJkyersrJS77//flFf98Od85NPPln017yjo0ONjY2aNWuWqqqqJEl+v18vv/yyurq6dNVVV2nr1q3y+Xz66U9/qhUrVuS54tFxpPN++umntWLFCkWjUblcLk2ePFltbW1atGhRniseHe3t7Vq5cqXC4bBcLpemTJmiH/zgB2ppaSnqn+9CU4rXgntscd9jpdK8z3KP5R57vPfYkg+dAAAAAADnlHT3WgAAAACAswidAAAAAADHEDoBAAAAAI4hdAIAAAAAHEPoBAAAAAA4htAJlKhnn31WS5YsyXcZAAAUHe6xwFCETgAAAACAYwidwDi0YcMGfexjH9OSJUt0yimnaO3atdq2bZsmT56sv//7v9cZZ5yhefPm6emnn85+zf33368FCxZo4cKFuvDCC7Vjx47se9/73ve0YMECLVq0SEuXLtXAwIAkKZFI6Prrr9eiRYs0b948vfLKK2N+rgAAjCXusUAeGADjyr59+8zixYvNzp07jTHG7Nq1yzQ1NZl169YZSeaee+4xxhizbt06U19fb/r6+sxbb71l6uvrTUdHhzHGmO985zvmggsuMMYYc88995ilS5eaUChkjDFm7969JpFImGeeecZ4PB6zYcMGY4wxd9xxhzn//PPH+nQBABgz3GOB/KClExhnXnzxRX344Yf61Kc+pZaWFp133nkyxigajcrn8+mqq66SJC1dulRTp07Vm2++qWeeeUYXXXSRZsyYIUm6/vrr9fTTT8sYo0cffVRf/epXVV1dLUmaMGGC3G63JOnEE0/MjjlZtmyZPvjggzycMQAAY4N7LJAfnnwXAGAoY4wWLlyo559/fsj+bdu2HfZ4y7JkjJFlWUP25SIQCGS33W63EonE8AsGAKBAcI8F8oOWTmCcWb58ubZs2TJkLMkbb7yhWCymWCymX/7yl5Kk9evXKxgMauHChTr33HP12GOPKRgMSpLuvPNOnXvuubIsS5dcconuuOMO9fb2SpJ6enqUTCbH/sQAAMgz7rFAftDSCYwzEyZM0O9+9zvddNNN+sY3vqF4PK6mpib967/+qyZNmqT3339fZ5xxhvr6+vTggw+qoqJC8+bN0z/90z/p/PPPlyQ1NjbqZz/7mSTpqquu0s6dO7Vs2TJ5vV6Vl5frqaeeyucpAgCQF9xjgfywjDEm30UAOLZt27ZpyZIl2r17d75LAQCgqHCPBZxF91oAAAAAgGNo6QQAAAAAOIaWTgAAAACAYwidAAAAAADHEDoBAAAAAI4hdAIAAAAAHEPoBAAAAAA4htAJAAAAAHAMoRMAAAAA4Jj/DyPjUZJYxIQXAAAAAElFTkSuQmCC\n"
     },
     "metadata": {}
    }
   ],
   "source": [
    "figure(num=None, figsize=(14, 6), dpi=80, facecolor='w', edgecolor='k')\n",
    "plt.subplot(121)\n",
    "line1, = plt.plot(bsi_train_loss)\n",
    "line2, = plt.plot(lrd_train_loss)\n",
    "plt.legend(handles=(line1, line2), labels=('Batch size increase', 'Learning rate decay'))\n",
    "plt.ylabel('Loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.title('Comparison of Loss by \\nLR decay and Batch size increasing')\n",
    "\n",
    "plt.subplot(122)\n",
    "line1, = plt.plot(bsi_test_acc)\n",
    "line2, = plt.plot(lrd_test_acc)\n",
    "plt.legend(handles=(line1, line2), labels=('Batch size increase', 'Learning rate decay'))\n",
    "plt.ylabel('Test acc')\n",
    "plt.xlabel('epoch')\n",
    "plt.title('Comparison of test acc by \\nLR decay and Batch size increasing')"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "2. 딥러닝개론.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4-final"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}