Network Slimming.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Network Slimming\n",
    "- Pruning via L1 on Batchnorm Gammas\n",
    "- Based on \"Learning Efficient Convolutional Networks through Network Slimming\"\n",
    "- CIFAR100 and VGG16\n",
    "- Soft (setting bn params to 0) and Hard (chaning architecture) pruning strategies\n",
    "- Depending on location 50% filter pruning results in 80% of weights pruned\n",
    "- lr schedule, weight init and hyperparams as in paper\n",
    "- For 50% pruning can't get same 73% perf. always 1% lower"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn, optim\n",
    "from torch.utils.data import DataLoader\n",
    "from torchvision import transforms\n",
    "from torchvision.models import vgg16_bn\n",
    "from torchvision.datasets import CIFAR100\n",
    "\n",
    "import math\n",
    "import copy\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import defaultdict\n",
    "\n",
    "DATASET_ROOT = \"/data/datasets/CIFAR100\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_transform = transforms.Compose(\n",
    "    [\n",
    "        transforms.RandomCrop(32, padding=4),\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.5071, 0.4867, 0.4408], [0.2675, 0.2565, 0.2761]),\n",
    "    ]\n",
    ")\n",
    "\n",
    "test_transform = transforms.Compose(\n",
    "    [\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.5071, 0.4867, 0.4408], [0.2675, 0.2565, 0.2761]),\n",
    "    ]\n",
    ")\n",
    "\n",
    "\n",
    "def get_loaders(batch_size=512):\n",
    "    train_ds = CIFAR100(root=DATASET_ROOT, transform=train_transform, train=True)\n",
    "    test_ds = CIFAR100(root=DATASET_ROOT, transform=test_transform, train=False)\n",
    "\n",
    "    train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True, num_workers=1, pin_memory=True)\n",
    "    test_loader = DataLoader(test_ds, batch_size=batch_size, shuffle=False, num_workers=1, pin_memory=True)\n",
    "\n",
    "    return train_loader, test_loader\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model():\n",
    "    model = vgg16_bn(pretrained=False)\n",
    "    model.features = model.features[:-1] # dropping the last maxpool\n",
    "    model.avgpool = nn.AvgPool2d(kernel_size=2)\n",
    "    model.classifier = nn.Sequential(nn.Linear(512, 100))\n",
    "    \n",
    "    for m in model.modules():\n",
    "        if isinstance(m, nn.Conv2d):\n",
    "            n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels\n",
    "            m.weight.data.normal_(0, math.sqrt(2. / n))\n",
    "            if m.bias is not None:\n",
    "                m.bias = None\n",
    "        elif isinstance(m, nn.BatchNorm2d):\n",
    "            m.weight.data.fill_(0.5)\n",
    "            m.bias.data.zero_()\n",
    "        elif isinstance(m, nn.Linear):\n",
    "            m.weight.data.normal_(0, 0.01)\n",
    "            m.bias.data.zero_()\n",
    "    \n",
    "    return model\n",
    "\n",
    "\n",
    "def get_bn_l1_loss(model):\n",
    "    \"\"\"L1 reg loss for bn gamma\"\"\"\n",
    "    loss = 0\n",
    "    for layer in model.features:\n",
    "        if isinstance(layer, nn.BatchNorm2d):\n",
    "            loss += layer.weight.abs().sum()\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model, loader) -> float:\n",
    "    \"\"\"Get Accuracy\"\"\"\n",
    "    correct = 0\n",
    "    n_obs = 0\n",
    "    \n",
    "    model.eval()\n",
    "    for X, y in loader:\n",
    "        X, y = X.cuda(), y.cuda()\n",
    "        with torch.no_grad():\n",
    "            out = model(X)\n",
    "        correct += (out.argmax(dim=1) == y).float().sum()\n",
    "        n_obs += len(X)\n",
    "    model.train()    \n",
    "    \n",
    "    accuracy = correct / n_obs\n",
    "    return accuracy.item()\n",
    "\n",
    "\n",
    "def train(model, n_epochs=160, lr=1e-1, batch_size=512, bn_l1_weight=1e-4):\n",
    "    \"\"\"\n",
    "    :param inital_lr: for finetuning to continue cosine annealing\n",
    "    :param max_epochs: will stop at n_epochs but max_epochs is needed for scheduling\n",
    "    :param last_epoch: when finetuning need to specify at what epoch previously stopped\n",
    "    \"\"\"\n",
    "    model = copy.deepcopy(model).cuda().train()\n",
    "    train_loader, test_loader = get_loaders(batch_size)\n",
    "    logs = defaultdict(list)\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    # Don't as me why but the original implementation uses l1 on top of the l2 for the whole model\n",
    "    optimiser = optim.SGD(model.parameters(), lr=lr, momentum=0.9, nesterov=True, weight_decay=1e-4)\n",
    "    \n",
    "    milestones = [int(n_epochs * .5), int(n_epochs * .75)]\n",
    "    print(f\"Reducing LR at: {milestones}\")\n",
    "    lr_scheduler = optim.lr_scheduler.MultiStepLR(optimiser, milestones=milestones, gamma=0.1)\n",
    "    \n",
    "    pbar = tqdm(range(n_epochs))\n",
    "    for epoch in pbar:\n",
    "        for X, y in train_loader:\n",
    "            X, y = X.cuda(), y.cuda()\n",
    "            out = model(X)\n",
    "            loss = criterion(out, y) + bn_l1_weight * get_bn_l1_loss(model)\n",
    "\n",
    "            optimiser.zero_grad()\n",
    "            loss.backward()\n",
    "            optimiser.step()\n",
    "\n",
    "        lr_scheduler.step()\n",
    "        #logs[\"train_acc\"].append(evaluate(model, train_loader))\n",
    "        logs[\"lr\"].append(optimiser.state_dict()[\"param_groups\"][0][\"lr\"])\n",
    "        logs[\"test_acc\"].append(evaluate(model, test_loader))\n",
    "        pbar.set_description(\"Test Acc: {:.4f}%\".format(logs['test_acc'][-1] * 100))\n",
    "    return model, logs\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/160 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reducing LR at: [80, 120]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Test Acc: 73.6600%: 100%|██████████| 160/160 [49:44<00:00, 18.65s/it]\n"
     ]
    }
   ],
   "source": [
    "model = get_model()\n",
    "trained_model, trained_logs = train(model, n_epochs=160, batch_size=64, bn_l1_weight=1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7365999817848206"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXicZ33v//dXo323Nlu2ZEne7STEjo2zkMTOvpDj0ALFoRAogUBLCimUkhTIr6U/OIe0JzScps0CPaRQSEMCxA0OISTEISR2bMdLbMeLLG+ybGvfRtKMRnOfP2akjKSxPXZkjWb0eV2XL+tZZuar257P3HM/9/M85pxDREQSX0q8CxARkbGhQBcRSRIKdBGRJKFAFxFJEgp0EZEkkRqvFy4pKXHV1dXxenkRkYS0efPmZudcabRtcQv06upqNm3aFK+XFxFJSGZ26GTbNOQiIpIkFOgiIklCgS4ikiQU6CIiSUKBLiKSJBToIiJJQoEuIpIk4jYPXUTkbASDDjMwMwA6+/oxQsu1jd0cbetlycxCphdmAdDjD/DK3iY6evv5HxdOJzv93cVeR28/B5u9VBfnkJ+VSltPP339A5TlZdA/4Nh1rIMUM86fUUCaZ3z7zAp0ERkXXl+Al/c00e3rp8c/wKGWHpq6fVQXZ1NVnENGagqeFCM1JYWBoKOjt589xztZX9dKty9AUU463b4A9W09ZKR6mFaQSUu3j7ae/qivV1kUCvTGTh++QBCA+3+9h0tmF1PX5KXXH6A4N4PinHSKc9Px+gY41tGLYeRkeMjJSCUzzUNzt4/GTh8ZaaG6dhztIBi+jUR6agr+8HN7UkIfMAPhjVlpHgqy0vD6A9SU5LByXiml4dC/dHYxC8vzx7yNLV43uFi2bJnTmaIiycM5x5ptDexv7CYjzYNzDn8giH/A0eb1s3bHMbr6AkP756R7KMnL4GhbL4Fg9BzKSvOwrHoKpXkZtHr95KSnUlGUha8/yLGOXopyMqguzsaTYgSCjuriHKYVZPLGgRa21XeQ7kmhKCedaxaUkepJ4V9frmXfiW7mTs0lNyOVVq+flm4/LV4/2ekeygsyMQOvbwCvL0Bv/wDFuemU5WXSPxBkIOhYVjWFRdPzOdTSQ3O3j/KCLDLTPDS095KSYpw/PZ9A0LHxYCteX4CsNA87GjrZcrht6IPg///A+Xzskqqzamcz2+ycWxZtm3roIu/SQNCREjEEMBm1ev189entvLDrxKht6Z4UMtJSuGp+GX968Uwqi7JJT02hOCcdM8MfCHK8o4/+YCgwAwMOT4pRmJ1GUU76WQ1bLK4sjLp+ec3yM36us3XzBeXDlrt9AXz9A6SmpJCZfm6GYhToIqcxEHTUNXXT1tPPlsNt/GLLUbLTPTy4egmBoOOOH25kxpQsHrt9Gd2+APc9s4Nb3jN91Bva6wvwzNYG3jrazicvq2H+tDz6+gfYcridXcc6SfcYf3xRBTkZsb8tO/v6yU1PJSUlfh8mXl+AjzzyOodaevj6+xfyqffV4B8IkmJGmsdO+0GXnprCzOLscao2fnIzUsk9g3/bs6EhF5FTePNwG1//xQ52HescWrdkZiH7G7tJTw2NqQ4EHV2+AJfNLqa+rZdDLT2kphg/+OR7GQgGefjlOo539tHY1Udff5A0j+EcrJhXyhsHWunyvTMMUZSTzqevqOH2S6vJzUilr3+AVq+fzr5+th/p4M3DbVQV57BkZiE/Wn+IX20/Rl5mKosrC7n90mquXVjGwZYeWr1+LppZGNO3hm1H2ikvzKQsL/OM28c5x1/+dAtr3zrGf3zqYi6fW3LGzyFn5lRDLgp0kSjeONDK939fxwtvn2BqXiZfuGYuVcXZVE7JZmZxNrWN3dzx+EYM+OGfLWfDgRa++vRbTMlO48HVS/hfz+1mz4kuBoKO6uJsLqwspCQ3g5svKKemJIdvr32bdXubWDGvlJvOn8YFFQUcae3l/7y0j5f3NFGYncbMomx2NXQOG1/Oy0wdGofOSvPw0Ytn0tc/wCv7mjjS2ktBVhodvaGDhDeeN427r5vLwWYvzsHK+WU4HC/sOkF+Vhor55Xy4w2H+cYvd5CV5uEzV9TwuZWzY54FcrS9l0fW7ec/Xj/E39w4n79YOWfM/x1kNAW6yBl4bX8zH31sA4XZaXz8kio+u2J21K/Kg7Mb0lNThh5XOSWbyqJsGrv6uPfpt7h0djGfuKz6jMaBtx1p5+F1+2nv6WfxzEKqirLJyUhl/rQ85pblcqyjj02H2rikpoiy/FCvOjAQ5JmtDbyyr4mlVVPo6gvw4G/34R8IDj1vTroHM6M7/I1g3tRc9p7o5qr5pWRnpPKr7ceYXZrDv/7pUmaV5rDneBdPba5n65F2PnPFLK5bNJUfvnaAZ7cfo63Hz9G2XhzwkWWVfPuPLojrsM9kokAXOQNPb67nyz/bxktfXsGs0tx4l3PWahu72HSwjQXl+fT4A/z3tgaCQfjji2ZQ1+zlX16q5aKqKfzvD19IemoKr9U284UnttLe48cROnaQ7kmhvDCTQy09FOWk0+r1s7RqCjOLsqkqzuZDSyuomJL8498TiWa5iJyBwR5sQVZanCt5d+aU5TGnLG9o+bLZ74xvXzyrmNXvrRw2xn7ZnBKe++IVPLJuP5lpHmpKcrh6QRl5man84NUDPL/zOP/04fdw9YKp4/p7SOwU6CIjDAZ6bmZyvz2iHTAtzcvg67csGrX+sytm89kVs8ejLHkXYhrYM7MbzWyPmdWa2T1Rtn/XzLaG/+w1s/axL1VkfHT7AqG506meeJcickZO2wUxMw/wEHAdUA9sNLM1zrldg/s45/4qYv+/BJacg1pFxkV3X4CcDIW5JJ5YeujLgVrnXJ1zzg88Adx6iv1vA346FsWJxEO3L5D0wy2SnGIJ9BnAkYjl+vC6UcysCqgBXjrJ9jvNbJOZbWpqajrTWkXGRbcvQG5GYh8QlckplkCPNrn0ZHMdVwNPOecGom10zj3qnFvmnFtWWloaa40i46q7L0CuhlwkAcUS6PVAZcRyBdBwkn1Xo+EWSXChHrqGXCTxxBLoG4G5ZlZjZumEQnvNyJ3MbD4wBXh9bEsUGV9eX+CMLpAlMlGcNtCdcwHgLuB54G3gSefcTjP7ppmtitj1NuAJF69TT0XGSJcvQJ4OikoCiul/rXNuLbB2xLr7Riz/3diVJRI/oTF0BbokHt0kWiTCQNDR2z+gIRdJSAp0kQhDp/0r0CUBKdBFIgwGusbQJREp0EUieMOBriEXSUQKdJEIg3cD0pCLJCIFukgEr4ZcJIEp0EUidGvIRRKYAl0kQreGXCSBKdBFIgzNctHVFiUBKdBFIrwz5KKrLUriUaCLROj2BchMSyHVo7eGJB79rxWJoJtbSCJToItE0M0tJJEp0EUi6H6iksgU6CIRdLciSWQKdJEIuha6JDIFukgE9dAlkSnQRSJ4NYYuCUyBLhKhSzeIlgSmQJe4qW/rYfOh1jF5rtrGbr71q100tPee9XP4A0H8gSB5CnRJUAp0AaCrr59HX9nPQNCd1eN/9PpBXt3XHPP+rV4/H3lkPZ9+fBPOnd1rRvrZpiM89vsDXPvAOn76xuFR24+299Lc7Tvlc3h1+zlJcAr0SabXP8BF//ACz25vGLb+vzYe4dtrd7O9vj2m51lf18LbxzoBONzSwzee2cnHfrCBu5/YMnQ9lJGOd/Txby/vZ0NdC1/46RaOtvfS1tNPW0//u/ulgP1N3VRMyWJReT73PbODwEBwaFub188t3/s9H31s/Sk/sHTpXEl0CvRJZn9TN61eP28eGh7cL+1uBOBE56l7sYO+/OQ2vvr0dgDW7WsC4KMXz+SXWxt4/LWDUR/zd2t28p1f7+Yjj67n1dpm3v+ecgAONHefza8yzP4mLxfMKODDyyroH3A0tPcNbbv/+T209fSz90T3qA+ySIN3K9LNLSRRKdAnmQPNXgAOt3qH1nX19fPGgdBYdmNXKAjXbGvgugfW0esfAGD38U7W17UAoV7+0fZettd3cKS1h1f2NlExJYtvfeB8zpuez7q9TaNed9uRdn698zifXTGLhz+2lAdXL+Yr188HoK7JO2r/aGobu3htfzOv7W8mGNHT9geCHG7tYXZpLlXFOQAcCv9+Ww638cTGw3zqfTUsmJbHd1/YO6z3Hul4Z2j8PT9L13KRxBRToJvZjWa2x8xqzeyek+zzJ2a2y8x2mtlPxrZMGSuDgX6opWdo3av7mgmEA7Ix3EPfeKCVfY3d/HLrUQaCjr/48Zvc/cRWAA62vBPAa7Y18FptM1fOK8XMuGJuKW8eahs17PKPz++hKCedv7x6LjeeP41bF8+gYkoWqSk2VNOglm4f9/96Nxf+/W/48MOvsfFgK3/z1DaufeAVPvrYBj762AZ+vOHQ0P6HW70MBB2zy3KoKs4e9vv9n5dqKcnN4K+um8uXr5/PwZYenn6zPmrb/HrHcXIzUrlo5pQzb1iRCeC0gW5mHuAh4CZgEXCbmS0asc9c4F7gfc6584C7z0GtchJbDrfxyLr9Me37Tg+9Z6iX+9LuRvIzUynJzeBEZ6iHPjhb5PHXDvLcjmPUNXs53tlHm9c/1KPOz0zl4XX78foHuHJuKQBXzishEHSs3x/qze9v6ubrv3yLV2ub+YuVs4cdcEz1pDCzOHtUoN/+72/wb+v2897qKdQ1efnww6/z1OZ6PrdiNk/ceQnLq4t48Lf7hj40ahtDj59dmsvUvEwyUlM4FP7QeetoB1fOLSUvM41rF5axZGYhD7ywlx7/8A8cfyDIr3cc5/pFU8lM08W5JDHF0kNfDtQ65+qcc37gCeDWEft8BnjIOdcG4JxrHNsy5VS+9+I+/udzuzne0XfafevC4ekLBGns8hEMOn63p4kr55UyvTCTE12hHnpDRx/pnhR2H+/i/3tmJ+mpof8qu493DY15f+Kyarr6AnhSjMvmFAOwtGoKWWkefr+vid/sPM51D6zjyY31/MmyCj52SdWoemaV5AwL9KYuHzsbOvnKDfP5/ifey0t/vZKv3DCfp//8Mu65aQGXzCrmb9+/kBavn8deqQv/TqF6akpySEkxZhZlc6ilh+ZuH01dPhaW5wFgZnz9/Qs50enj0fBjB71a20RnX4BbLiyPveFFJphYAn0GcCRiuT68LtI8YJ6Z/cHM1pvZjdGeyMzuNLNNZrapqWn0OKtE19Hbz+v7W4aNGw/q8Qf4Q7g3vG7vqT9HnXMcaOpmZtHgsISXnQ2dNHf7uGp+GWV5mTRG9NBXLZ5OQVYaLV4/d187F4A9xzupa/IyLT+TD15UAcDSmVPIzwyNO2ekerhkVhEv7DrB3zy9nfOmF/DavVdz/4cujNrzrQkH+uDvNjgv/ZJZoQ+Igqw0Pn/VHJZEDIMsrizk5gum8djv62ju9rG/0cvU/AzywjVUFYcCffexLgAWlucPPXZpVRHvf085j6yrG/o2AvDstmMUZKVx+ZzSU7ahyEQWS6BblHUjkyUVmAusBG4Dvm9mhaMe5NyjzrllzrllpaV645xOjz/AZ/5jE0v/4QVue2w9P9t8ZNQ+v9/XjD8QxJNivLzn1B+SrV4/nX0BVs4Ptf2h1h42HAh9GLxvTgll+Rk0dvno8Qfo6O2npiSHz1xRw6LyfD5zxSyKctLZfbyLumYvs0pzqC7J4Y7La7jjipphr3PF3FIaOvrwB4I8uHoxJbkZJ62ppiQXXyBIQ0doiGfTwTYyUlM4f3rBKX+XL103nx7/AD9ef4j9Td3MLs0d2lZVnMPh1p6haZULpuUNe+w9Ny5gIOj4t5dDw1R9/QP8ZtcJbjhv6tA3EZFEFMv/3nqgMmK5Ahg596seeMY51++cOwDsIRTw8i783z8c5IVdJ/jEZdXMKcvlh68dGnUSzm93nSAvM5UPLJ7Bq/ua6T/JDA54Z/z88jkleFKMwy09bDzYSmVRFtMKMpmal0mr1z90QHFGYRZ3XT2XtV+8gjRPCgum5fH2sU7qmrqpKQnNJvnGLYu44bxpw17nmoVlZKSm8M1bz2dWRNBGM/g8g7VtPNTGhRWFpw3WOWW5XDW/lB+vP8z+xpGBnk1v/wCv7GuiLC+D4hEfKJVF2dx0wTR+/mY9vf4Bntl6lG5fgA8sGfnFUySxxBLoG4G5ZlZjZunAamDNiH1+CVwFYGYlhIZg6pCz1ub18/DL+7l2YRnfuGURd1xew9vHOtl8qG1on4Gg46XdjVw1v4zrFpXR5QvwZsT2kQbHz+dNzWN6YSaHWnvYdLCN91YXAVCWHwq+wZOLygsyhz1+/rQ8djZ00tkXOGVQVxXnsOPvb+BDSytO+3vOKn0n0Hv9A+w82sHS6thmmfzZ+2po7vbR5QsMPQ8wNKT0+v6WYcMtkW5bPpPOvgDPbm/g0VfqWFSez6XhYR6RRHXaQHfOBYC7gOeBt4EnnXM7zeybZrYqvNvzQIuZ7QJ+B3zFOddyropOVkfbe/nSf23la794i7/77510+wP89Q2hudq3Lp5OfmYqj78emq7nXCjMW7x+rl00lcvmlJCaYlHngA860OwlNcWomJJFVVEOf6htpsXrZ3k40KeGA33rkVCgTy/MGvb4hdPyh6Y3zirJ4VTSYrzJclleBtnpHuqavGyrbycQdCyrii3Qr5hbwpyy0AdLZA+9OjwXPRB0LCjPi/rYi2uKmFWSw7fWvs3+Ji+fXTELs2ijiyKJI6ZT4pxza4G1I9bdF/GzA74U/iNn4dntDdz79FtDgdnbP8AfLZnBgmmhHmZ2eiofXlbJD187SG1jN01dPpq7fRRmp7FiXin5mWlcVDWFp9+s5/I5JVw2p2TouQeHaQ42e5lZnD00XfDV2tC1V5bXhHvoeaEe+ZbD7ZjBtCg99EGRPeJ3w8yoKclhy+E2PCmhQF0aY6CbGX++Yjb3/uKtYT3xGVOy8KQYA0HHwmnRe+hmxurllXx77W5mFGZx8wWa3SKJT+c4TwCNXX18+cltLCzP53url1CSl86GutZRQw+fvqKGw609OAcLy/O4pKaYlfNLKQif2fjVGxfwxSe28NHvb+DahWV84Zq57Dnexf98bjcFWWl09QVYXBk62FgVHpYoyU0fGsceHHLZe6KLsryMUb3seVPzMCPcy88es9//hvOm8cALe9lW38GcslwKs9NjfuwHl1Zww/nThs1vT/OkML0wkyOtvScdcgH44EUV/MtLtXz+qjkxf6MQmcgU6OfI3hNdfOtXb/PdjyymKOfUAfX93x+gfyDIP39kMTPDZzpetaBs1H7lBVk8dvuykz7P0qop/PZLK/jBqwd4ZN1+Vv3LH4bW9/UPcKDZy9ypoV724Djze6uLhoYainMySDEIutBrjZSV7qGmODTXe7A3PRa+cM1cbr5gGs9uP3ba2S3RRLs6YnVxDic6fKf8JlGcm8Hmb1ynMJekoUA/Rx5/7SDr9jbx8Lr9/O3NC0dtf3lPI4daerj+vKn8eP0hVl04nerTjEvHIjPNw+evmsPtl1bx5KZ6CrPS+KMlMzCDNw+3M3dqaKx58LUGD4gCeFKM0rwMTnT6mFE4OtABPrdy9uhJq2NgTlked18bfbz7bNzynnJmleScNqwV5pJMFOjnQP9AkLVvHSPF4D9eP8inr6gZGp+G0CySO3+0GX8gyLfWvk3/QJDPXzVnTGvIy0zjjsuHzw+PHJteMC2PB/7kQm48f/iUw7K8TE50+kbNcBn0J8sqo66faD7y3pnxLkFk3Kl7cg68uq+Ztp5+/vbmhfQPOP71d+9cZ6W528fnfrSZ0twMHvn4Ui6sKODjl1QNDYWMFzPjjy+qIDt9+Gf64EyXkTNcRGTiUw/9HFizrYH8zFQ+fmkV+05085MNh7nj8hoqi7L5xi930Oz18/TnLuOCioJRJ+XEW2n4m8T0wug9dBGZuNRDH2O9/gF+s/M4N19QTkaqh7uvm0tKCnzn17vZdLCV53Yc5/Mr53BBxZkf/BsP6qGLJC710MfYS7sb8foHWHXhdCA0W+TOK2fzvRf3sb2+g7K8DD5zZc1pniV+Fpbnk5PuoapobOaZi8j4UQ99jD2z9ShleRlcHHEa+edWzKIsL4PDrT186bp5o8atJ5LrF01l8zeuoyBbd+0RSTQTN1kSUEdvPy/vaeJjl1QNm6ednZ7KP334Qp7bcZwPT/BZImamGzyIJCgF+hh6fudx/ANBVi2ePmrblfNKuXKeLhksIueOhlzOQjDoot5s4r+3NVBVnM2FE/SAp4gkNwX6GdpQ18Lyb/+Wf35x37D1tY3d/KG2mVUXTtdV+0QkLhToZ+Dnb9bzsR9soLnbz5qtR4fWH+vo5fYfbKAoJ53blusMRRGJDwV6jI609nDPz99iadUUvnzdPA629HCw2Utf/wC3/+ANuvoC/PDPlmv+tojEjQ6Kxug7v95NisF3P7IYfyDI/35hLy/vaSQ7I5V9jd18//ZlnD9DY+ciEj8K9FN48Lf7eHlvI1fMKeHZ7cf4wjVzhy4rW1OSw8t7mzje0ceCaXlcs3D05W5FRMaTAv0kjrT28C+/20dmmocth9uZmp/B51bMGtq+Yl4pj79+EOfgOx+8QAdCRSTuFOgn8eCL+zAzfvNXV3Ki00deZuqwMzxXzC/lh68dpCgnnVsX627xIhJ/kz7QO3r7h27hNmhXQyc/f7OeT72vhvKCrKh377l0VjFFOel86n3VOrNSRCaESR3obxxo5aOPrefFL6+gqjh0o+Kv/WIHu451kpuRyp+vnH3Sx2amefjDV68mM00ThURkYpjUgV7b2E0g6Nh9vIuq4hzWbGugtqmbr9wwn5svKKc4N+OUj89KV89cRCaOSR3orV4fEDoAOvh3TXHOmN8OTkRkPMQ0XmBmN5rZHjOrNbN7omz/pJk1mdnW8J9Pj32pY6+52w/A4XCgH2rpYWZxdjxLEhE5a6ftoZuZB3gIuA6oBzaa2Rrn3K4Ru/6Xc+6uc1DjOdPqfSfQg0HH4dYeVuiKiCKSoGLpoS8Hap1zdc45P/AEcOu5LWt8tEQMuTR2+fAFglSphy4iCSqWQJ8BHIlYrg+vG+mDZrbdzJ4ys6h3cTCzO81sk5ltampqOotyx1ZLeMjlSFsvB1u8AMws1q3XRCQxxRLo0U6BHHkx8P8Gqp1z7wF+Czwe7Ymcc48655Y555aVlsZ/aKPF6yc1xfAHgmw80ApAVZF66CKSmGIJ9HogssddATRE7uCca3HO+cKLjwFLx6a8cycYdLR5/SwozwPg1dpmPCnGjCm6WqKIJKZYAn0jMNfMaswsHVgNrIncwczKIxZXAW+PXYnnRmdfP4GgY0nlFADePNzG9MJM0jw6UUhEEtNpZ7k45wJmdhfwPOAB/t05t9PMvglscs6tAb5gZquAANAKfPIc1jwmWsIzXC6oKMAM+gccVUUaPxeRxBXTiUXOubXA2hHr7ov4+V7g3rEt7dwaPCBaXpBJeX4mDR19moMuIglt0o4vDJ4lWpSTTmX4QKgOiIpIIpt0gb77eCfBoBs6S7QkN4OZg4GuHrqIJLBJFehHWnu46cHf88y2o0NniU7JTh8K9Er10EUkgU2qi3MdbPHiHLxxoI10j5GfmUp6agrXLJzKzoZO5pTlxrtEEZGzNql66Mfa+wDYeqSdFq9/6PK4i6bn8/DHl5KRqsvhikjimlQ99IaOXgD2nugiIzWFopz0OFckIjJ2JmUPfSDo2F7fTrECXUSSyKQK9IaOXirCp/YHHRTnKtBFJHlMqkA/1tHH+dMLqCwKhXpxzqlvMScikkgmTaA75zjW3kt5YSaLw9dv0Ri6iCSTSRPonX0BvP4BphdksbiyENCQi4gkl0kzy+VYeIZLeWEmc8pySU0xzTsXkaQyeQI9PMOlvCCLBdPy2f5315OdPml+fRGZBCbNkMvgHPTphZkACnMRSTqTJtCPtffhSTHK8jLjXYqIyDkxaQK9oaOXqXkZeFKi3SJVRCTxTZpAP9bex/RC3S9URJLXpAn0ho5eyhXoIpLEJkWgO+c41tHH9AKNn4tI8poUgV7X7MUfCOqeoSKS1CZFoL/49gkAVs4vi3MlIiLnzqQI9Bd2nWBReT4zNIYuIkks6QO9pdvH5kNtXLdoarxLERE5p5I+0F/a3UjQoUAXkaQXU6Cb2Y1mtsfMas3snlPs9yEzc2a2bOxKfHde2HWC8oJMzpueH+9SRETOqdMGupl5gIeAm4BFwG1mtijKfnnAF4ANY13k2XrurWP8bk8j1y+aipnOEBWR5BZLD305UOucq3PO+YEngFuj7PcPwP1A3xjWd9Z+tP4Qf/GTN3lPRSF3Xzsv3uWIiJxzsQT6DOBIxHJ9eN0QM1sCVDrnnj3VE5nZnWa2ycw2NTU1nXGxserrH+Bbv9rF5XNK+M9PX8wU3ZlIRCaBWAI92liFG9polgJ8F/jy6Z7IOfeoc26Zc25ZaWlp7FWeoQ0HWunrD/Kpy2vITPOcs9cREZlIYgn0eqAyYrkCaIhYzgPOB142s4PAJcCaeB4YfXlPIxmpKVw6qzheJYiIjLtYAn0jMNfMaswsHVgNrBnc6JzrcM6VOOeqnXPVwHpglXNu0zmpOAbr9jRxyaxi9c5FZFI5baA75wLAXcDzwNvAk865nWb2TTNbda4LPFOHW3qoa/aycv65G9IREZmIYroPm3NuLbB2xLr7TrLvyndf1tl7eW8jAFfpui0iMskk3Zmi6/Y0UV2cTXVJTrxLEREZV0kX6HXNXs6fURDvMkRExl3SBXpLt49izTsXkUkoqQLdHwjS2RegKCcj3qWIiIy7pAr0th4/AEW56qGLyOSTVIHe0h0K9BINuYjIJJRUgd7qDffQFegiMgklVaC3eH0AFGvIRUQmoaQK9Hd66DooKiKTT9IFeopBYVZavEsRERl3SRXozd1+pmSnk5KiuxOJyOSTVIHe6vVp/FxEJq0kC3S/ZriIyKSVVIHe4vVTrAOiIjJJJVWgq4cuIpNZ0gR6/0CQ9p5+BbqITFpJE+iD13HRQVERmaySJtAHTyrSGLqITFbJE+jduo6LiExuSRPoLV4NuYjI5JY0ga4rLYrIZJc0gd7S7Uk7DoUAAAqQSURBVMMMpmQr0EVkckqeQPf6KcxKw6PruIjIJBVToJvZjWa2x8xqzeyeKNs/Z2ZvmdlWM3vVzBaNfamnppOKRGSyO22gm5kHeAi4CVgE3BYlsH/inLvAObcYuB94YMwrPY26Ji+VRdnj/bIiIhNGLD305UCtc67OOecHngBujdzBOdcZsZgDuLEr8fQ6+/rZ29jFksop4/myIiITSmoM+8wAjkQs1wMXj9zJzD4PfAlIB66O9kRmdidwJ8DMmTPPtNaT2n6kA+fgoqrCMXtOEZFEE0sPPdpRxlE9cOfcQ8652cBXga9HeyLn3KPOuWXOuWWlpaVnVukpbDnchhlcWKlAF5HJK5ZArwcqI5YrgIZT7P8E8IF3U9SZ2nKknTmlueRn6tZzIjJ5xRLoG4G5ZlZjZunAamBN5A5mNjdi8f3AvrEr8dScc2w53MaSmeqdi8jkdtoxdOdcwMzuAp4HPMC/O+d2mtk3gU3OuTXAXWZ2LdAPtAGfOJdFRzrY0kNbTz8XzdQBURGZ3GI5KIpzbi2wdsS6+yJ+/uIY1xWzLYfbAFiiQBeRSS7hzxTdeqSd3IxU5pTlxrsUEZG4SvhAP9rWy8yibJ3yLyKTXsIHelO3j9I83dRCRCThA725y0dJrgJdRCShA905R3O3Xz10ERESPNA7ewP4B4IKdBEREjzQm7r7ACjRbedERBI80LtCt51TD11EJNEDvdsHQKkOioqIJHagN3eFA109dBGRxA70pm4faR6jIEtXWRQRSexAD89BN9NZoiIiCR3ozTpLVERkSEIHepPOEhURGZLQgd7c7dMMFxGRsIQN9GAwdNp/SZ5OKhIRgQQO9PbefgaCTj10EZGwhA30pvAc9BIdFBURARI40Jt1lqiIyDAJG+hNOktURGSYhA90DbmIiIQkbKA3e32kp6aQl5Ea71JERCaEhA30dm8/RdnpOu1fRCQscQO9109hti7KJSIyKKZAN7MbzWyPmdWa2T1Rtn/JzHaZ2XYze9HMqsa+1OHae/p1lUURkQinDXQz8wAPATcBi4DbzGzRiN22AMucc+8BngLuH+tCR+roVaCLiESKpYe+HKh1ztU55/zAE8CtkTs4537nnOsJL64HKsa2zNHae/o15CIiEiGWQJ8BHIlYrg+vO5k7gOeibTCzO81sk5ltampqir3KKEJj6LqOi4jIoFgCPdo0Ehd1R7OPAcuAf4y23Tn3qHNumXNuWWlpaexVjtDXP0Bff1BDLiIiEWKZxF0PVEYsVwANI3cys2uBrwErnHO+sSkvuo7efgANuYiIRIilh74RmGtmNWaWDqwG1kTuYGZLgEeAVc65xrEvc7j2nnCgZ2nIRURk0GkD3TkXAO4CngfeBp50zu00s2+a2arwbv8I5AI/M7OtZrbmJE83JtRDFxEZLabz5p1za4G1I9bdF/HztWNc1ym19/gBNIYuIhIhIc8UbVcPXURklIQM9I7BMXRNWxQRGZKQgd7e6yc1xchJ98S7FBGRCSMxAz18lqiutCgi8o7EDHRdx0VEZJSEDPQOXWlRRGSUhAx0XcdFRGS0xAz0nn4K1UMXERkmIQO9o6efAs1BFxEZJuECvX8gSJcvoOu4iIiMkHCB3qmzREVEokq4QNdp/yIi0SVeoIdP+9e0RRGR4RIu0Dt6Q1da1LRFEZHhEi7Q37m5hXroIiKREjfQNYYuIjJMwgV6xZQsrl80lbxMBbqISKSY7lg0kVx/3jSuP29avMsQEZlwEq6HLiIi0SnQRUSShAJdRCRJKNBFRJKEAl1EJEko0EVEkoQCXUQkSSjQRUSShDnn4vPCZk3AobN8eAnQPIbljKWJWpvqOjMTtS6YuLWprjN3NrVVOedKo22IW6C/G2a2yTm3LN51RDNRa1NdZ2ai1gUTtzbVdebGujYNuYiIJAkFuohIkkjUQH803gWcwkStTXWdmYlaF0zc2lTXmRvT2hJyDF1EREZL1B66iIiMoEAXEUkSCRfoZnajme0xs1ozuyeOdVSa2e/M7G0z22lmXwyvLzKzF8xsX/jvKXGqz2NmW8zs2fByjZltCNf1X2YWl7tsm1mhmT1lZrvDbXfpRGgzM/ur8L/jDjP7qZllxqPNzOzfzazRzHZErIvaPhbyvfB7YbuZXRSH2v4x/G+53cx+YWaFEdvuDde2x8xuGM+6Irb9tZk5MysJL49bm52sLjP7y3Cb7DSz+yPWv/v2cs4lzB/AA+wHZgHpwDZgUZxqKQcuCv+cB+wFFgH3A/eE198DfCdO9X0J+AnwbHj5SWB1+OeHgT+PU12PA58O/5wOFMa7zYAZwAEgK6KtPhmPNgOuBC4CdkSsi9o+wM3Ac4ABlwAb4lDb9UBq+OfvRNS2KPz+zABqwu9bz3jVFV5fCTxP6ATGkvFus5O011XAb4GM8HLZWLbXuL1pxqiBLgWej1i+F7g33nWFa3kGuA7YA5SH15UDe+JQSwXwInA18Gz4P29zxBtvWDuOY1354eC0Eevj2mbhQD8CFBG6LeOzwA3xajOgekQIRG0f4BHgtmj7jVdtI7b9EfCf4Z+HvTfDwXrpeNYFPAVcCByMCPRxbbMo/5ZPAtdG2W9M2ivRhlwG33iD6sPr4srMqoElwAZgqnPuGED477I4lPTPwN8AwfByMdDunAuEl+PVbrOAJuD/hoeDvm9mOcS5zZxzR4F/Ag4Dx4AOYDMTo83g5O0z0d4PnyLU+4U412Zmq4CjzrltIzbFu83mAVeEh/LWmdl7x7KuRAt0i7IurvMuzSwXeBq42znXGc9awvXcAjQ65zZHro6yazzaLZXQV9B/c84tAbyEhhDiKjwmfSuhr7rTgRzgpii7TrQ5vhPl3xUz+xoQAP5zcFWU3calNjPLBr4G3Bdtc5R149lmqcAUQsM9XwGeNDMbq7oSLdDrCY2LDaoAGuJUC2aWRijM/9M59/Pw6hNmVh7eXg40jnNZ7wNWmdlB4AlCwy7/DBSaWWp4n3i1Wz1Q75zbEF5+ilDAx7vNrgUOOOeanHP9wM+By5gYbQYnb58J8X4ws08AtwB/6sLjBXGubTahD+dt4fdBBfCmmU2Lc12EX//nLuQNQt+iS8aqrkQL9I3A3PDsg3RgNbAmHoWEP1V/ALztnHsgYtMa4BPhnz9BaGx93Djn7nXOVTjnqgm1z0vOuT8Ffgd8KF51hWs7Dhwxs/nhVdcAu4hzmxEaarnEzLLD/66DdcW9zcJO1j5rgNvDMzcuAToGh2bGi5ndCHwVWOWc64nYtAZYbWYZZlYDzAXeGI+anHNvOefKnHPV4fdBPaEJDMeJf5v9klAnCzObR2hiQDNj1V7n6mDAOTzIcDOhGSX7ga/FsY7LCX0l2g5sDf+5mdB49YvAvvDfRXGscSXvzHKZFf4PUgv8jPBR9jjUtBjYFG63XxL6+hn3NgP+HtgN7AB+RGi2wbi3GfBTQuP4/YSC6I6TtQ+hr+kPhd8LbwHL4lBbLaGx38H3wMMR+38tXNse4KbxrGvE9oO8c1B03NrsJO2VDvw4/P/sTeDqsWwvnfovIpIkEm3IRURETkKBLiKSJBToIiJJQoEuIpIkFOgiIklCgS4ikiQU6CIiSeL/Ab/i5nvNl7SCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(trained_logs[\"test_acc\"])\n",
    "trained_logs[\"test_acc\"][-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAWjklEQVR4nO3dfZBd9X3f8ffnXiHxYJ4RGEtyJRucjjxxaqxgu82DaxIbEgc1DUyEPRMypYP7QJ/c1IFxS2yaP4qbhDQ104QJZAh2DJQ6qQYroU5IM1OPSxB2DMhYsYKJkbGNCARMXB4kffvHPSutr1f3XEl39+65fr9mdnTPOb+7+92f9n727O/8zu+mqpAkza7etAuQJC0ug16SZpxBL0kzzqCXpBln0EvSjFsx7QKGnXHGGbV+/fpplyFJnfLAAw88VVWrFzq27IJ+/fr1bN++fdplSFKnJPnLQx1z6EaSZpxBL0kzzqCXpBln0EvSjDPoJWnGjRX0SS5MsjPJriRXL3D8h5J8NsneJJcMHbs8yZeaj8snVbgkaTytQZ+kD9wIXARsBC5LsnGo2VeAnwV+Z+i5pwG/ALwZOB/4hSSnHn3ZkqRxjTOP/nxgV1U9CpDkdmAz8IW5BlX1WHNs/9Bz3wl8qqqebo5/CrgQ+PhRVz7khZf38eRzL0760+q7XAJrTjmOXi/TLkU6YuME/Rrg8XnbuxmcoY9joeeuGfO5h2Xn17/J5hs/vRifWt/l/tnbXsv7L/zb0y5DOmLjBP1CpzLjvlvJWM9NciVwJcCrX/3qMT/1t1t32vH88qXfd0TPlQ7luru/wFPP+5eium2coN8NrJu3vRZ4YszPvxt429Bz//dwo6q6CbgJYNOmTUf0llennbCSn3rT2iN5qnRIv/y/drJveEBS6phxZt3cD5ybZEOSlcAWYOuYn/8e4B1JTm0uwr6j2Sd1Qq8XfLtNdV1r0FfVXuAqBgH9CHBnVe1Icl2SiwGSfH+S3cClwG8k2dE892ngPzL4ZXE/cN3chVmpC3oJ+wx6ddxYq1dW1TZg29C+a+c9vp/BsMxCz70FuOUoapSmpt8L+/Yb9Oo274yVRugFPKFX1xn00gi9eEav7jPopRH6Pcfo1X0GvTRCL866UfcZ9NIIvR4O3ajzDHpphH6COa+uM+ilEZKw36EbdZxBL43gPHrNAoNeGqHvGb1mgEEvjZDAfhc1U8cZ9NIIzqPXLDDopRH6PYdu1H0GvTRCEvZ7MVYdZ9BLI/SD8+jVeQa9NILTKzULDHppBG+Y0iww6KURnEevWWDQSyO4qJlmgUEvjdBzUTPNAINeGsF59JoFBr00gm8lqFlg0EsjDN5hatpVSEfHoJdG6HsxVjPAoJdG6MVFzdR9Br00Qq/nm4Or+wx6aYS+F2M1Awx6aYReHKNX9xn00giDoZtpVyEdHYNeGsGLsZoFBr00gnfGahYY9NIIvcQ3B1fnGfTSCL3g0I06z6CXRnDoRrNgrKBPcmGSnUl2Jbl6geOrktzRHL8vyfpm/zFJbk3yUJJHklwz2fKlxTW31o03TanLWoM+SR+4EbgI2AhclmTjULMrgGeq6hzgBuD6Zv+lwKqq+l7gTcB7534JSF3QSwDn0qvbxjmjPx/YVVWPVtVLwO3A5qE2m4Fbm8d3ARckCVDACUlWAMcBLwHPTaRyaQn0m1eIOa8uGyfo1wCPz9ve3exbsE1V7QWeBU5nEPp/A3wN+ArwS1X19PAXSHJlku1Jtu/Zs+ewvwlpsaQ5o3ecXl02TtBngX3DP/WHanM+sA94FbAB+LdJXvMdDatuqqpNVbVp9erVY5QkLY1+z6Ebdd84Qb8bWDdvey3wxKHaNMM0JwNPA+8G/qCqXq6qJ4FPA5uOtmhpqfQ9o9cMGCfo7wfOTbIhyUpgC7B1qM1W4PLm8SXAvTWYpvAV4O0ZOAF4C/DFyZQuLb4m571pSp3WGvTNmPtVwD3AI8CdVbUjyXVJLm6a3QycnmQX8D5gbgrmjcArgIcZ/ML4rap6cMLfg7Ro5oZuPKNXl60Yp1FVbQO2De27dt7jFxhMpRx+3vML7Ze64sAYvUGvDvPOWGmEA7NuvBirDjPopREOXoydciHSUTDopRHmbphy6EZdZtBLIzh0o1lg0EsjOI9es8Cgl0bozQ3deEavDjPopRF6XozVDDDopRG8YUqzwKCXRnA9es0Cg14aoefFWM0Ag14a4cDQjYuaqcMMemmEJue9YUqdZtBLI/S8GKsZYNBLI/S9M1YzwKCXRnAevWaBQS+N4J2xmgUGvTSC0ys1Cwx6aQTvjNUsMOilEbwzVrPAoJdGmJtH7xm9usygl0bwzljNAoNeGuHA0I1n9Oowg14aYS7oy6BXhxn00ghzQzf7HLpRhxn00gguaqZZYNBLI8wtaubQjbrMoJdG6DuPXjPAoJdG8IYpzQKDXhphblEzR27UZQa9NMKBWTcmvTrMoJdGcPVKzQKDXhqh5ztMaQaMFfRJLkyyM8muJFcvcHxVkjua4/clWT/v2BuSfCbJjiQPJTl2cuVLi+vgDVMGvbqrNeiT9IEbgYuAjcBlSTYONbsCeKaqzgFuAK5vnrsC+CjwT6rq9cDbgJcnVr20yA6uXjndOqSjMc4Z/fnArqp6tKpeAm4HNg+12Qzc2jy+C7ggSYB3AA9W1ecBquqvqmrfZEqXFl/PNx7RDBgn6NcAj8/b3t3sW7BNVe0FngVOB14HVJJ7knw2yfsX+gJJrkyyPcn2PXv2HO73IC0a59FrFowT9Flg3/BP/aHarAB+AHhP8+9PJrngOxpW3VRVm6pq0+rVq8coSVoa/QOzbqZciHQUxgn63cC6edtrgScO1aYZlz8ZeLrZ/ydV9VRVfQvYBpx3tEVLS2XuhimHbtRl4wT9/cC5STYkWQlsAbYOtdkKXN48vgS4twarQN0DvCHJ8c0vgB8GvjCZ0qXF5/RKzYIVbQ2qam+SqxiEdh+4pap2JLkO2F5VW4GbgduS7GJwJr+lee4zSX6FwS+LArZV1ScX6XuRJq7vO0xpBrQGPUBVbWMw7DJ/37XzHr8AXHqI536UwRRLqXMOzLrxjF4d5p2xUotevBirbjPopRb9Xhy6UacZ9FKLJA7dqNMMeqlFP3F6pTrNoJda9Hth3/5pVyEdOYNeapF4w5S6zaCXWvR7Dt2o2wx6qUU/cVEzdZpBL7VI4jx6dZpBL7Xo97wzVt1m0Est+vGGKXWbQS+1iPPo1XEGvdSi3/POWHWbQS+16AX2mfPqMINeatFzHr06zqCXWvRd1EwdZ9BLLXpejFXHGfRSi56LmqnjDHqpRb/nombqNoNeauHQjbrOoJda9FzUTB1n0EstXKZYXWfQSy16gf1ejFWHGfRSi56LmqnjDHqpRS+hDHp1mEEvtRi8ObhBr+4y6KUWvV5c1EydZtBLLXrBoRt1mkEvtfDNwdV1Br3UoucYvTrOoJdaDIZupl2FdOQMeqlFv+c8enWbQS+18M3B1XVjBX2SC5PsTLIrydULHF+V5I7m+H1J1g8df3WS55P83GTKlpaO7zClrmsN+iR94EbgImAjcFmSjUPNrgCeqapzgBuA64eO3wD8/tGXKy09h27UdeOc0Z8P7KqqR6vqJeB2YPNQm83Arc3ju4ALkgQgyT8AHgV2TKZkaWnFRc3UceME/Rrg8Xnbu5t9C7apqr3As8DpSU4Afh740KgvkOTKJNuTbN+zZ8+4tUtLou8YvTpunKDPAvuGf+oP1eZDwA1V9fyoL1BVN1XVpqratHr16jFKkpaOa92o61aM0WY3sG7e9lrgiUO02Z1kBXAy8DTwZuCSJB8GTgH2J3mhqj5y1JVLS2Qw62baVUhHbpygvx84N8kG4KvAFuDdQ222ApcDnwEuAe6tweIgPzjXIMkHgecNeXWNbw6urmsN+qram+Qq4B6gD9xSVTuSXAdsr6qtwM3AbUl2MTiT37KYRUtLyTF6dd04Z/RU1TZg29C+a+c9fgG4tOVzfPAI6pOmLi5qpo7zzlipRb/nDVPqNoNeatELXoxVpxn0Uoued8aq4wx6qYVr3ajrDHqpRc9ZN+o4g15q0esNbpjyfWPVVQa91KI/WJ/PC7LqLINeatFrVnJy+EZdZdBLLXpN0nvTlLrKoJda9Jug94ReXWXQSy3mhm6cS6+uMuilFr04dKNuM+ilFnNB7/RKdZVBL7XoezFWHWfQSy0OzLrxjF4dZdBLLeYuxprz6iqDXmrR92KsOs6gl1rMDd14Z6y6yqCXWszNutm/f8qFSEfIoJda9JtXiRdj1VUGvdTiwBm9Qa+OMuilFgeHbgx6dZNBL7XoO49eHWfQSy0OrEfvxVh1lEEvtXCMXl1n0Est+s6jV8cZ9FILlylW1xn0UgvvjFXXGfRSi/6BMfopFyIdIYNeanHgrQRNenWUQS+1ODB0Y9Crowx6qUXPoRt13FhBn+TCJDuT7Epy9QLHVyW5ozl+X5L1zf4fTfJAkoeaf98+2fKlxeeiZuq61qBP0gduBC4CNgKXJdk41OwK4JmqOge4Abi+2f8U8BNV9b3A5cBtkypcWireMKWuG+eM/nxgV1U9WlUvAbcDm4fabAZubR7fBVyQJFX1uap6otm/Azg2yapJFC4tFRc1U9eNE/RrgMfnbe9u9i3Ypqr2As8Cpw+1+Sngc1X14vAXSHJlku1Jtu/Zs2fc2qUlcWBRM4NeHTVO0GeBfcM/8SPbJHk9g+Gc9y70BarqpqraVFWbVq9ePUZJ0tLxYqy6bpyg3w2sm7e9FnjiUG2SrABOBp5uttcCvwv8TFX9xdEWLC21XvMqcYxeXTVO0N8PnJtkQ5KVwBZg61CbrQwutgJcAtxbVZXkFOCTwDVV9elJFS0tpb5r3ajjWoO+GXO/CrgHeAS4s6p2JLkuycVNs5uB05PsAt4HzE3BvAo4B/gPSf6s+Thz4t+FtIhc60Zdt2KcRlW1Ddg2tO/aeY9fAC5d4Hm/CPziUdYoTZXTK9V13hkrtTiwqJnvMKWOMuilFplb1MwzenWUQS+16LuomTrOoJdaHHwrwSkXIh0hg15q4dCNus6gl1r0XetGHWfQSy36zqNXxxn0Uot4Z6w6zqCXWsyd0XtCr64y6KUWPS/GquMMeqlFz6EbdZxBL7U4OHRj0KubDHqpxcEz+ikXIh0hg15q4Ri9um6sZYql72ZJ6AU++eAT7Hrym9MuZ9k7+bhj+ODFr2fViv60S1HDoJfG8ONveBVf/NpzfOkbz0+7lGXtWy/t46t//f+4dNM6znv1qdMuRw2DXhrDf73sjdMuoRN2PPEsP/5r/4cnn3th2qVoHsfoJU3MWScdC8A3nntxypVoPoNe0sScdvxKjumHr3tGv6wY9JImptcLZ554LN8w6JcVg17SRJ150iqedOhmWTHoJU3UWSce69DNMmPQS5qos05a5dDNMmPQS5qos04+lm++sJdvvbR32qWoYdBLmqizTnSK5XJj0EuaqINz6R2+WS4MekkT9cqTVwEG/XJi0EuaqDM9o192DHpJE3XiqhUcd0zfMfplxKCXNFFJnGK5zBj0kiburJOO9e7YZcSglzRxZ53k3bHLievRS5q4uaGbF/fuI2Ta5XRGL7CiP/nz77GCPsmFwH8B+sBvVtV/Gjq+Cvht4E3AXwE/XVWPNceuAa4A9gH/sqrumVj1kpalV558HC/u3c/3/Ps/mHYpnfKuN5zNR9593sQ/b2vQJ+kDNwI/CuwG7k+ytaq+MK/ZFcAzVXVOki3A9cBPJ9kIbAFeD7wK+MMkr6uqfZP+RiQtH//wjWvYu28/e/f7huqH47WrX7Eon3ecM/rzgV1V9ShAktuBzcD8oN8MfLB5fBfwkSRp9t9eVS8CX06yq/l8n5lM+ZKWo1NPWMl7f/i10y5DjXGCfg3w+Lzt3cCbD9WmqvYmeRY4vdn/f4eeu2b4CyS5Eriy2Xw+yc6xql/YGcBTR/H8xWJdh2e51gXLtzbrOnzLtbYjqetvHerAOEG/0JWU4b/HDtVmnOdSVTcBN41RS6sk26tq0yQ+1yRZ1+FZrnXB8q3Nug7fcq1t0nWNc3l3N7Bu3vZa4IlDtUmyAjgZeHrM50qSFtE4QX8/cG6SDUlWMri4unWozVbg8ubxJcC9VVXN/i1JViXZAJwL/OlkSpckjaN16KYZc78KuIfB9MpbqmpHkuuA7VW1FbgZuK252Po0g18GNO3uZHDhdi/wz5dgxs1EhoAWgXUdnuVaFyzf2qzr8C3X2iZaVwYn3pKkWeUSCJI04wx6SZpxMxP0SS5MsjPJriRXT7GOdUn+OMkjSXYk+VfN/tOSfCrJl5p/T51Sff0kn0tyd7O9Icl9TV13NBfcp1HXKUnuSvLFpu/euhz6LMm/af4fH07y8STHTqvPktyS5MkkD8/bt2AfZeDXmtfDg0kmf1/96Lr+c/N/+WCS301yyrxj1zR17UzyzqWsa96xn0tSSc5otqfaX83+f9H0yY4kH563/+j7q6o6/8HgIvFfAK8BVgKfBzZOqZazgfOaxycCfw5sBD4MXN3svxq4fkr1vQ/4HeDuZvtOYEvz+NeBfzqlum4F/nHzeCVwyrT7jMHNfV8GjpvXVz87rT4Dfgg4D3h43r4F+wj4MeD3GdzL8hbgviWu6x3Aiubx9fPq2ti8PlcBG5rXbX+p6mr2r2MwueQvgTOWSX/9feAPgVXN9pmT7K8le9Es5gfwVuCeedvXANdMu66mlv/JYJ2gncDZzb6zgZ1TqGUt8EfA24G7mx/qp+a9IL+tH5ewrpOaQM3Q/qn2GQfv+D6NwQy1u4F3TrPPgPVDAbFgHwG/AVy2ULulqGvo2E8CH2sef9trswncty5lXQyWafk+4LF5QT/V/mJw8vAjC7SbSH/NytDNQss0fMdSC0styXrgjcB9wFlV9TWA5t8zp1DSrwLvB/Y326cDf11Ve5vtafXba4A9wG81w0q/meQEptxnVfVV4JeArwBfA54FHmB59NmcQ/XRcnpN/CMGZ8sw5bqSXAx8tao+P3Ro2v31OuAHmyHBP0ny/ZOsa1aCfqylFpZSklcA/wP411X13DRraep5F/BkVT0wf/cCTafRbysY/Cn736rqjcDfMBiGmKpmvHszgz+ZXwWcAFy0QNPlOEd5WfzfJvkAg3toPja3a4FmS1JXkuOBDwDXLnR4gX1L2V8rgFMZDBv9O+DOJJlUXbMS9MtqqYUkxzAI+Y9V1Sea3d9IcnZz/GzgySUu6+8BFyd5DLidwfDNrwKnNMtWwPT6bTewu6rua7bvYhD80+6zHwG+XFV7qupl4BPA32V59NmcQ/XR1F8TSS4H3gW8p5pxhynX9VoGv7Q/37wO1gKfTfLKKddF8/U/UQN/yuCv7jMmVdesBP04yzQsiea38M3AI1X1K/MOzV8m4nIGY/dLpqquqaq1VbWeQf/cW1XvAf6YwbIVU6mrqe3rwONJvqfZdQGDu6mn2mcMhmzekuT45v91rq6p99k8h+qjrcDPNLNJ3gI8OzfEsxQyeLOinwcurqpvDdU7lWVRquqhqjqzqtY3r4PdDCZOfJ0p9xfwewxOvkjyOgYTEp5iUv21WBcblvqDwVXzP2dwVfoDU6zjBxj8afUg8GfNx48xGA//I+BLzb+nTbHGt3Fw1s1rmh+cXcB/p7nqP4Wa/g6wvem332PwZ+zU+wz4EPBF4GHgNgazH6bSZ8DHGVwreJlBSF1xqD5i8Cf/jc3r4SFg0xLXtYvB2PLca+DX57X/QFPXTuCipaxr6PhjHLwYO+3+Wgl8tPk5+yzw9kn2l0sgSNKMm5WhG0nSIRj0kjTjDHpJmnEGvSTNOINekmacQS9JM86gl6QZ9/8BVEF1abEUVs8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(trained_logs[\"lr\"])\n",
    "plt.ylim(bottom=0)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_gammas(model, cat=False):\n",
    "    \"\"\"\n",
    "    Get the gamma parameters of the bn layers\n",
    "    :param cat: whether to concatenate and all gammas into one tensor.\n",
    "    \"\"\"\n",
    "    res = [bn.weight.view(-1).detach().cpu() for bn in model.modules() if isinstance(bn, nn.BatchNorm2d)]\n",
    "    if cat:\n",
    "        res = torch.cat(res)\n",
    "    return res\n",
    "\n",
    "\n",
    "def get_prune_masks(model, percentage=0.5, min_filters_per_layer: int=1) -> list:\n",
    "    \"\"\"Reuturns a list of masks of which filters to keep\"\"\"\n",
    "    res = []\n",
    "    threshold = np.quantile(get_gammas(model, cat=True).abs(), percentage)\n",
    "    bns = [layer for layer in model.modules() if isinstance(layer, nn.BatchNorm2d)]\n",
    "    for bn in bns:\n",
    "        local_threshold = threshold\n",
    "        # leave at least min_filters_per_layer\n",
    "        if (bn.weight.abs() >= local_threshold).float().sum() < min_filters_per_layer:\n",
    "            local_threshold = bn.weight.abs().sort()[0][-min_filters_per_layer].item()\n",
    "        mask = (bn.weight.abs() >= local_threshold).float().detach()  # filters to keep\n",
    "        res.append(mask)\n",
    "    return res\n",
    "\n",
    "\n",
    "def count_filters(model, cat=True):\n",
    "    gammas = get_gammas(model, cat=False)\n",
    "    counts = [(l != 0).int().sum().item() for l in gammas]\n",
    "    if cat == True:\n",
    "        counts = sum(counts) \n",
    "    return counts\n",
    "\n",
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Soft Pruning\n",
    "- Just setting the BN params to 0, so the whole channel output will be 0 for that filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prune_soft(model, percentage=0.5, min_filters_per_layer: int=1):\n",
    "    \"\"\"Just set bn gamma and beta to 0 to produce 'unused' filters\"\"\"\n",
    "    model = copy.deepcopy(model.cpu())\n",
    "    masks = get_prune_masks(model, percentage, min_filters_per_layer)#[:-1]\n",
    "    bns = [layer for layer in model.modules() if isinstance(layer, nn.BatchNorm2d)]#[:-1]\n",
    "    for mask, bn in zip(masks, bns):\n",
    "        with torch.no_grad():\n",
    "            bn.weight.mul_(mask)\n",
    "            bn.bias.mul_(mask)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pruned Filters: 0.49644886363636365\n",
      "Architecture: [30, 64, 118, 128, 253, 255, 255, 417, 181, 5, 5, 5, 411]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABIoAAAEvCAYAAAAq+CoPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dfZSdVX0v8O+vCZIKWCikyiVwk0uDvBslQFxBxXLFgCyxXWLBqkhdxFrsba8tivder62VtayiLXgBG0sEuxCQF2tWV1oF68tVQUlqpLxKIhGmsCSAUCjCLWHfP+YJDslkcpKZOTOTfD5rnTXn7LOf5/mdWZsZ5pu991OttQAAAADAL010AQAAAABMDoIiAAAAAJIIigAAAADoCIoAAAAASCIoAgAAAKAjKAIAAAAgSTJ9ogvYkr322qvNnj17ossAAAAA2G6sXLnyodbazI3bJ31QNHv27KxYsWKiywAAAADYblTVT4Zrt/QMAAAAgCSCIgAAAAA6giIAAAAAkkyBPYoAAACAqe0//uM/MjAwkKeeemqiS9nhzJgxI7NmzcpOO+3UU39BEQAAADCuBgYGsttuu2X27NmpqokuZ4fRWsvDDz+cgYGBzJkzp6djLD0DAAAAxtVTTz2VPffcU0jUZ1WVPffcc6tmcgmKAAAAgHEnJJoYW/t9FxQBAAAAkMQeRQAAAECffeF7947p+d569H5jer7x8I1vfCPnnXde/v7v/35Mznf22Wdn+fLlOfHEE7Nw4cIccMABOfjgg0d9XkERAAAAQJL169dn2rRpE11GT/76r/8669aty84775x3vvOdOemkk8YkKLL0DAAAANjurV27NgceeGBOP/30HH744Xnzm9+cJ598MrNnz85HPvKRHHPMMbn66qtz7LHHZsWKFUmShx56KLNnz06SXHrppfmt3/qtLFq0KHPnzs373//+58791a9+Na985Svzile8IqecckqeeOKJJMk//uM/5sADD8wxxxyT6667bsT6vvnNb2bevHmZN29eXv7yl+fxxx9Pay1nn312Dj300Bx22GG56qqrkiRvfOMb8+///u85+uij82d/9mdZtmxZzj777MybNy9r1qwZ1ffJjKJJotdpd1NhOh0AAABMRnfddVcuueSSLFy4ML/7u7+biy66KEkyY8aMfPvb306SfOYzn9ns8atWrcoPfvCD7LzzznnpS1+aP/iDP8gv//Iv56Mf/WhuuOGG7LLLLvmLv/iLfOpTn8r73//+nHnmmfmnf/qn/Pqv/3p++7d/e8TazjvvvFx44YVZuHBhnnjiicyYMSPXXXddVq1alR/+8Id56KGHcuSRR+bVr351li1bll133TWrVq1Kktxzzz056aST8uY3v3nU3yMzigAAAIAdwr777puFCxcmSd72trc9Fw5tKcTZ4Ljjjsuv/MqvZMaMGTn44IPzk5/8JDfddFNuv/32LFy4MPPmzctll12Wn/zkJ7nzzjszZ86czJ07N1WVt73tbSOee+HChXnf+96XCy64II8++mimT5+eb3/72znttNMybdq0vPjFL85rXvOa3HzzzaP7JmyBGUUAAADADmHjW8VveL3LLrs81zZ9+vQ8++yzSZKnnnrqef133nnn555PmzYtzzzzTFpred3rXpcrrrjieX1XrVq1VbemP+ecc/KGN7why5cvz4IFC3LDDTektdbz8WPFjCIAAABgh3DvvffmxhtvTJJcccUVOeaYYzbpM3v27KxcuTJJcs0112zxnAsWLMh3vvOdrF69Okny5JNP5kc/+lEOPPDA3HPPPc/tGbRxkLSxNWvW5LDDDssHPvCBzJ8/P3feeWde/epX56qrrsr69euzbt26fOtb38pRRx21ybG77bZbHn/88S3W2gszigAAAIC+mqj9dw866KBcdtllefe73525c+fmPe95Tz796U8/r8+f/Mmf5C1veUv+9m//Nr/xG7+xxXPOnDkzl156aU477bQ8/fTTSZKPfvSjOeCAA7JkyZK84Q1vyF577ZVjjjkmt95662bP81d/9Vf5+te/nmnTpuXggw/OCSeckBe84AW58cYb87KXvSxVlY9//ON5yUtessmxp556as4888xccMEFueaaa7L//vtv5XfmF2oipjFtjfnz57cNu41vz2xmDQAAwPbqjjvuyEEHHTShNaxduzYnnXTSiGHN9mq4739VrWytzd+4r6VnAAAAACSx9AwAAADYAcyePXtSzCb63Oc+l/PPP/95bQsXLsyFF144QRU93xaDoqraN8nnk7wkybNJlrTWzq+qX01yVZLZSdYmeUtr7Wc1uKX3+UlOTPJkkne21v65O9fpSf5Xd+qPttYuG9uPAwAAADB5nXHGGTnjjDMmuozN6mXp2TNJ/ri1dlCSBUnOqqqDk5yT5GuttblJvta9TpITksztHouTXJwkXbD04SRHJzkqyYerao8x/CwAAAAAjMIWg6LW2gMbZgS11h5PckeSfZKcnGTDjKDLkrype35yks+3QTcl2b2q9k7y+iTXt9Yeaa39LMn1SRaN6acBAAAAYJtt1WbWVTU7ycuTfC/Ji1trDySDYVKSX+u67ZPkviGHDXRtm2sHAAAAYBLoOSiqql2TXJvkj1pr/zZS12Ha2gjtw11rcVWtqKoV69at67VEAAAAgE08+uijueiii7b6uBNPPDGPPvromNSw6667jsl5xltPdz2rqp0yGBJd3lq7rmv+aVXt3Vp7oFta9mDXPpBk3yGHz0pyf9d+7Ebt3xjueq21JUmWJMn8+fOHDZMAAACAKWrF58b2fPNH3hx6Q1D0+7//+89rX79+faZNm7bZ45YvXz4m5U0lW5xR1N3F7JIkd7TWPjXkrWVJTu+en57ky0Pa31GDFiR5rFua9pUkx1fVHt0m1sd3bQAAAADj5pxzzsmaNWsyb968HHnkkXnta1+bt771rTnssMOSJG9605tyxBFH5JBDDsmSJUueO2727Nl56KGHsnbt2hx00EE588wzc8ghh+T444/Pz3/+8yTJmjVrsmjRohxxxBF51atelTvvvDNJcs899+SVr3xljjzyyHzoQx/q/4feRr0sPVuY5O1JfqOqVnWPE5N8LMnrquruJK/rXifJ8iQ/TrI6yWeT/H6StNYeSfLnSW7uHh/p2gAAAADGzcc+9rHsv//+WbVqVT7xiU/k+9//fs4999zcfvvtSZKlS5dm5cqVWbFiRS644II8/PDDm5zj7rvvzllnnZXbbrstu+++e6699tokyeLFi/PpT386K1euzHnnnffcrKU//MM/zHve857cfPPNeclLXtK/DztKW1x61lr7dobfXyhJjhumf0ty1mbOtTTJ0q0pEAAAAGAsHXXUUZkzZ85zry+44IJ86UtfSpLcd999ufvuu7Pnnns+75g5c+Zk3rx5SZIjjjgia9euzRNPPJHvfve7OeWUU57r9/TTTydJvvOd7zwXJr397W/PBz7wgXH9TGOlpz2KAAAAALYXu+yyy3PPv/GNb+SGG27IjTfemBe+8IU59thj89RTT21yzM477/zc82nTpuXnP/95nn322ey+++5ZtWrVsNcZ3M1naun5rmcAAAAAU9Fuu+2Wxx9/fNj3Hnvsseyxxx554QtfmDvvvDM33XRTz+d90YtelDlz5uTqq69OkrTW8sMf/jBJsnDhwlx55ZVJkssvv3yUn6B/BEUAAADAdm3PPffMwoULc+ihh+bss89+3nuLFi3KM888k8MPPzwf+tCHsmDBgq069+WXX55LLrkkL3vZy3LIIYfky18evNfX+eefnwsvvDBHHnlkHnvssTH7LOOtBrcUmrzmz5/fVqxYMdFljLsvfO/envq99ej9xrkSAAAAGFt33HFHDjrooIkuY4c13Pe/qla21uZv3NeMIgAAAACSCIoAAAAA6AiKAAAAAEgiKAIAAAD6YLLvkby92trvu6AIAAAAGFczZszIww8/LCzqs9ZaHn744cyYMaPnY6aPYz0AAAAAmTVrVgYGBrJu3bqJLmWHM2PGjMyaNavn/oIiAAAAYFzttNNOmTNnzkSXQQ8sPQMAAAAgiaAIAAAAgI6gCAAAAIAkgiIAAAAAOoIiAAAAAJIIigAAAADoCIoAAAAASCIoAgAAAKAjKAIAAAAgiaAIAAAAgI6gCAAAAIAkPQRFVbW0qh6sqluHtF1VVau6x9qqWtW1z66qnw957zNDjjmiqv6lqlZX1QVVVePzkQAAAADYFtN76HNpkv+T5PMbGlprv73heVV9MsljQ/qvaa3NG+Y8FydZnOSmJMuTLEryD1tfMgAAAADjYYszilpr30ryyHDvdbOC3pLkipHOUVV7J3lRa+3G1lrLYOj0pq0vFwAAAIDxMto9il6V5KettbuHtM2pqh9U1Ter6lVd2z5JBob0GejaAAAAAJgkell6NpLT8vzZRA8k2a+19nBVHZHk76rqkCTD7UfUNnfSqlqcwWVq2W+//UZZIgAAAAC92OYZRVU1PclvJblqQ1tr7enW2sPd85VJ1iQ5IIMziGYNOXxWkvs3d+7W2pLW2vzW2vyZM2dua4kAAAAAbIXRLD37r0nubK09t6SsqmZW1bTu+X9JMjfJj1trDyR5vKoWdPsavSPJl0dxbQAAAADG2BaDoqq6IsmNSV5aVQNV9a7urVOz6SbWr05yS1X9MMk1SX6vtbZhI+z3JPmbJKszONPIHc8AAAAAJpEt7lHUWjttM+3vHKbt2iTXbqb/iiSHbmV9AAAAAPTJaO96BgAAAMB2QlAEAAAAQBJBEQAAAAAdQREAAAAASQRFAAAAAHQERQAAAAAkERQBAAAA0BEUAQAAAJBEUAQAAABAR1AEAAAAQBJBEQAAAAAdQREAAAAASQRFAAAAAHQERQAAAAAkERQBAAAA0BEUAQAAAJBEUAQAAABAR1AEAAAAQBJBEQAAAAAdQREAAAAASQRFAAAAAHQERQAAAAAk6SEoqqqlVfVgVd06pO1Pq+pfq2pV9zhxyHsfrKrVVXVXVb1+SPuirm11VZ0z9h8FAAAAgNHoZUbRpUkWDdP+l621ed1jeZJU1cFJTk1ySHfMRVU1raqmJbkwyQlJDk5yWtcXAAAAgEli+pY6tNa+VVWzezzfyUmubK09neSeqlqd5KjuvdWttR8nSVVd2fW9fasrBgAAAGBcjGaPovdW1S3d0rQ9urZ9ktw3pM9A17a5dgAAAAAmiW0Nii5Osn+SeUkeSPLJrr2G6dtGaB9WVS2uqhVVtWLdunXbWCIAAAAAW2ObgqLW2k9ba+tba88m+Wx+sbxsIMm+Q7rOSnL/CO2bO/+S1tr81tr8mTNnbkuJAAAAAGylbQqKqmrvIS9/M8mGO6ItS3JqVe1cVXOSzE3y/SQ3J5lbVXOq6gUZ3PB62baXDQAAAMBY2+Jm1lV1RZJjk+xVVQNJPpzk2Kqal8HlY2uTvDtJWmu3VdUXM7hJ9TNJzmqtre/O894kX0kyLcnS1tptY/5pAAAAANhmvdz17LRhmi8Zof+5Sc4dpn15kuVbVR0AAAAAfTOau54BAAAAsB0RFAEAAACQRFAEAAAAQEdQBAAAAEASQREAAAAAHUERAAAAAEkERQAAAAB0BEUAAAAAJBEUAQAAANARFAEAAACQRFAEAAAAQEdQBAAAAEASQREAAAAAHUERAAAAAEkERQAAAAB0BEUAAAAAJBEUAQAAANARFAEAAACQRFAEAAAAQEdQBAAAAEASQREAAAAAHUERAAAAAEl6CIqqamlVPVhVtw5p+0RV3VlVt1TVl6pq9659dlX9vKpWdY/PDDnmiKr6l6paXVUXVFWNz0cCAAAAYFv0MqPo0iSLNmq7PsmhrbXDk/woyQeHvLemtTave/zekPaLkyxOMrd7bHxOAAAAACbQFoOi1tq3kjyyUdtXW2vPdC9vSjJrpHNU1d5JXtRau7G11pJ8Psmbtq1kAAAAAMbDWOxR9LtJ/mHI6zlV9YOq+mZVvapr2yfJwJA+A13bsKpqcVWtqKoV69atG4MSAQAAANiSUQVFVfU/kzyT5PKu6YEk+7XWXp7kfUm+UFUvSjLcfkRtc+dtrS1prc1vrc2fOXPmaEoEAAAAoEfTt/XAqjo9yUlJjuuWk6W19nSSp7vnK6tqTZIDMjiDaOjytFlJ7t/WawMAAAAw9rZpRlFVLUrygSRvbK09OaR9ZlVN657/lwxuWv3j1toDSR6vqgXd3c7ekeTLo64eAAAAgDGzxRlFVXVFkmOT7FVVA0k+nMG7nO2c5PruLvc3dXc4e3WSj1TVM0nWJ/m91tqGjbDfk8E7qP1yBvc0GrqvEQAAAAATbItBUWvttGGaL9lM32uTXLuZ91YkOXSrqgMAAACgb8birmcAAAAAbAcERQAAAAAkERQBAAAA0BEUAQAAAJBEUAQAAABAR1AEAAAAQBJBEQAAAAAdQREAAAAASQRFAAAAAHQERQAAAAAkERQBAAAA0BEUAQAAAJBEUAQAAABAR1AEAAAAQBJBEQAAAAAdQREAAAAASQRFAAAAAHQERQAAAAAkERQBAAAA0BEUAQAAAJBEUAQAAABAR1AEAAAAQJIeg6KqWlpVD1bVrUPafrWqrq+qu7uve3TtVVUXVNXqqrqlql4x5JjTu/53V9XpY/9xAAAAANhWvc4oujTJoo3azknytdba3CRf614nyQlJ5naPxUkuTgaDpSQfTnJ0kqOSfHhDuAQAAADAxOspKGqtfSvJIxs1n5zksu75ZUneNKT9823QTUl2r6q9k7w+yfWttUdaaz9Lcn02DZ8AAAAAmCCj2aPoxa21B5Kk+/prXfs+Se4b0m+ga9tc+yaqanFVraiqFevWrRtFiQAAAAD0ajw2s65h2toI7Zs2traktTa/tTZ/5syZY1ocAAAAAMMbTVD0025JWbqvD3btA0n2HdJvVpL7R2gHAAAAYBIYTVC0LMmGO5ednuTLQ9rf0d39bEGSx7qlaV9JcnxV7dFtYn181wYAAADAJDC9l05VdUWSY5PsVVUDGbx72ceSfLGq3pXk3iSndN2XJzkxyeokTyY5I0laa49U1Z8nubnr95HW2sYbZAMAAAAwQXoKilprp23mreOG6duSnLWZ8yxNsrTn6gAAAADom/HYzBoAAACAKUhQBAAAAEASQREAAAAAHUERAAAAAEkERQAAAAB0BEUAAAAAJBEUAQAAANARFAEAAACQRFAEAAAAQEdQBAAAAEASQREAAAAAHUERAAAAAEkERQAAAAB0BEUAAAAAJBEUAQAAANARFAEAAACQRFAEAAAAQEdQBAAAAEASQREAAAAAHUERAAAAAEkERQAAAAB0tjkoqqqXVtWqIY9/q6o/qqo/rap/HdJ+4pBjPlhVq6vqrqp6/dh8BAAAAADGwvRtPbC1dleSeUlSVdOS/GuSLyU5I8lfttbOG9q/qg5OcmqSQ5L8pyQ3VNUBrbX121oDAAAAAGNnrJaeHZdkTWvtJyP0OTnJla21p1tr9yRZneSoMbo+AAAAAKM0VkHRqUmuGPL6vVV1S1Utrao9urZ9ktw3pM9A1wYAAADAJDDqoKiqXpDkjUmu7pouTrJ/BpelPZDkkxu6DnN428w5F1fViqpasW7dutGWCAAAAEAPxmJG0QlJ/rm19tMkaa39tLW2vrX2bJLP5hfLywaS7DvkuFlJ7h/uhK21Ja21+a21+TNnzhyDEgEAAADYkrEIik7LkGVnVbX3kPd+M8mt3fNlSU6tqp2rak6SuUm+PwbXBwAAAGAMbPNdz5Kkql6Y5HVJ3j2k+eNVNS+Dy8rWbnivtXZbVX0xye1JnklyljueAQAAAEweowqKWmtPJtlzo7a3j9D/3CTnjuaaAAAAAIyPsbrrGQAAAABTnKAIAAAAgCSjXHrGVljxuRHf3v/eR5Ika/Y7pR/VAAAAAGzCjCIAAAAAkgiKAAAAAOgIigAAAABIIigCAAAAoCMoAgAAACCJoAgAAACAjqAIAAAAgCSCIgAAAAA6giIAAAAAkgiKAAAAAOgIigAAAABIIigCAAAAoCMoAgAAACCJoAgAAACAjqAIAAAAgCSCIgAAAAA6giIAAAAAkgiKAAAAAOgIigAAAABIMgZBUVWtrap/qapVVbWia/vVqrq+qu7uvu7RtVdVXVBVq6vqlqp6xWivDwAAAMDYGKsZRa9trc1rrc3vXp+T5GuttblJvta9TpITksztHouTXDxG1wcAAABglMZr6dnJSS7rnl+W5E1D2j/fBt2UZPeq2nucagAAAABgK4xFUNSSfLWqVlbV4q7txa21B5Kk+/prXfs+Se4bcuxA1wYAAADABJs+BudY2Fq7v6p+Lcn1VXXnCH1rmLa2SafBwGlxkuy3335jUCIAAAAAWzLqGUWttfu7rw8m+VKSo5L8dMOSsu7rg133gST7Djl8VpL7hznnktba/Nba/JkzZ462RAAAAAB6MKqgqKp2qardNjxPcnySW5MsS3J61+30JF/uni9L8o7u7mcLkjy2YYkaAAAAABNrtEvPXpzkS1W14VxfaK39Y1XdnOSLVfWuJPcmOaXrvzzJiUlWJ3kyyRmjvD4AAAAAY2RUQVFr7cdJXjZM+8NJjhumvSU5azTXBAAAAGB8jMVdzwAAAADYDgiKAAAAAEgiKAIAAACgIygCAAAAIImgCAAAAICOoAgAAACAJIIiAAAAADqCIgAAAACSCIoAAAAA6AiKAAAAAEgiKAIAAACgIygCAAAAIImgCAAAAICOoAgAAACAJIIiAAAAADqCIgAAAACSCIoAAAAA6AiKAAAAAEgiKAIAAACgIygCAAAAIImgCAAAAICOoAgAAACAJKMIiqpq36r6elXdUVW3VdUfdu1/WlX/WlWruseJQ475YFWtrqq7qur1Y/EBAAAAABgb00dx7DNJ/ri19s9VtVuSlVV1fffeX7bWzhvauaoOTnJqkkOS/KckN1TVAa219aOoAQAAAIAxss0zilprD7TW/rl7/niSO5LsM8IhJye5srX2dGvtniSrkxy1rdcHAAAAYGyNyR5FVTU7ycuTfK9rem9V3VJVS6tqj65tnyT3DTlsICMHSwAAAAD00aiDoqraNcm1Sf6otfZvSS5Osn+SeUkeSPLJDV2HObxt5pyLq2pFVa1Yt27daEsEAAAAoAejCoqqaqcMhkSXt9auS5LW2k9ba+tba88m+Wx+sbxsIMm+Qw6fleT+4c7bWlvSWpvfWps/c+bM0ZQIAAAAQI9Gc9ezSnJJkjtaa58a0r73kG6/meTW7vmyJKdW1c5VNSfJ3CTf39brAwAAADC2RnPXs4VJ3p7kX6pqVdf2P5KcVlXzMrisbG2SdydJa+22qvpiktszeMe0s9zxDAAAAGDy2OagqLX27Qy/79DyEY45N8m523pNAAAAAMbPmNz1DAAAAICpT1AEAAAAQBJBEQAAAAAdQREAAAAASQRFAAAAAHQERQAAAAAkERQBAAAA0BEUAQAAAJBEUAQAAABAR1AEAAAAQJJk+kQXwBS04nO99Zt/xvjWAb0aZsx+755HNmlbs98pm7S99ej9xqUkAACAyUhQxPgRKAEAAMCUIijiF3oNdqbCdYVPbIP97716k7bv3dv78UfP+dXnN/Q6DoWqAADAJCEoAvpjlGHIFzZKbIYLdSbaJsvZ7vnkqM63SfA0kcY6zBKOAQDApCQoAkZnrGeEbeZ8+9+76Z5CjI+NQ7mk9+//mvWbHtvrPk/D7hs1ivMBAABbT1C0I5ioJWUTyWyFKWO4cIBBfZmh1If/BkYTPAEAAP0lKALYQQwbyg0TPu0/imuM9ZLAYc83bRRL8oTDAAAwIkHRVLYjzhSif4wvJqleZ6FNqj2eAABgihAUsWPbEZeoCYDYQdjzCAAAtp6gCHox2QMl4Q/0ZKuWsm303/Nwey0Nd87NzmTangJnAAC2W4KiSWaL+3uMZm8OADax2aVsG+3f1OveTZs7n9lMAABMBYIiGEuTfeYRAAAAjKDvQVFVLUpyfpJpSf6mtfaxftcAU4YlZbDdGG7G6GZWs/VkzX6nbNLW6wyl4ZbRjeaOdZbbAQBsP/oaFFXVtCQXJnldkoEkN1fVstba7f2sAybcDhAA9XpnKmDbjCZ46nUZXa96XW632SBro5+Jo/35sXGINpoAbThTeslgj9/rbf0eAgBTX79nFB2VZHVr7cdJUlVXJjk5iaAIALYzG4dZo5lB1Y/r9rwPVY/n2+xMq43PNwmD9U2CyB43fR/RkJBqLD7z0DBLkAUAY6ffQdE+Se4b8nogydF9roEd2Hj8z3ivfwgAsGOZjAHQtup10/d+GhpmjUUIOdxyzi1dd6xsuLbAC4DJoN9BUQ3T1jbpVLU4yeLu5RNVdde4VjU57JXkoYkuArbAOGWyM0aZCozTSelPJvzavzOBFQzDOGUqME6ZCibzOP3PwzX2OygaSLLvkNezkty/cafW2pIkS/pV1GRQVStaa/Mnug4YiXHKZGeMMhUYp0wFxilTgXHKVDAVx+kv9fl6NyeZW1VzquoFSU5NsqzPNQAAAAAwjL7OKGqtPVNV703ylSTTkixtrd3WzxoAAAAAGF6/l56ltbY8yfJ+X3cK2KGW2jFlGadMdsYoU4FxylRgnDIVGKdMBVNunFZrm+wlDQAAAMAOqN97FAEAAAAwSQmK+qyqFlXVXVW1uqrOGeb9navqqu7971XV7P5XyY6shzH6vqq6vapuqaqvVdWwt1SE8bSlcTqk35urqlXVlLrTBNuHXsZpVb2l+5l6W1V9od81Qg+/9/erqq9X1Q+63/0nTkSd7LiqamlVPVhVt27m/aqqC7oxfEtVvaLfNUIP4/R3uvF5S1V9t6pe1u8at4agqI+qalqSC5OckOTgJKdV1cEbdXtXkp+11n49yV8m+Yv+VsmOrMcx+oMk81trhye5JsnH+1slO7oex2mqarck/y3J9/pbIfQ2TqtqbpIPJlnYWjskyR/1vVB2aD3+PP1fSb7YWnt5Bu9YfFF/q4RcmmTRCO+fkGRu91ic5OI+1AQbuzQjj9N7krym+xvqzzPJ9y0SFPXXUUlWt9Z+3Fr7f0muTHLyRn1OTnJZ9/yaJMdVVWNzzNAAAANISURBVPWxRnZsWxyjrbWvt9ae7F7elGRWn2uEXn6WJoO/hD+e5Kl+FgedXsbpmUkubK39LElaaw/2uUboZZy2JC/qnv9Kkvv7WB+ktfatJI+M0OXkJJ9vg25KsntV7d2f6mDQlsZpa+27G37fZwr8DSUo6q99ktw35PVA1zZsn9baM0keS7JnX6qD3sboUO9K8g/jWhFsaovjtKpenmTf1trf97MwGKKXn6cHJDmgqr5TVTdV1Uj/EgnjoZdx+qdJ3lZVAxm8c/Ef9Kc06NnW/v8rTLRJ/zfU9IkuYAcz3MygjW8710sfGC89j7+qeluS+UleM64VwaZGHKdV9UsZXLr7zn4VBMPo5efp9AwulTg2g/+y+H+r6tDW2qPjXBts0Ms4PS3Jpa21T1bVK5P8bTdOnx3/8qAn/n5iyqiq12YwKDpmomsZiRlF/TWQZN8hr2dl0+m7z/WpqukZnOI70lRLGEu9jNFU1X9N8j+TvLG19nSfaoMNtjROd0tyaJJvVNXaJAuSLLOhNX3W6+/8L7fW/qO1dk+SuzIYHEG/9DJO35Xki0nSWrsxyYwke/WlOuhNT///ChOtqg5P8jdJTm6tPTzR9YxEUNRfNyeZW1VzquoFGdwQcNlGfZYlOb17/uYk/9Rak4jTL1sco92Snr/OYEhkPw0mwojjtLX2WGttr9ba7Nba7AyuA39ja23FxJTLDqqX3/l/l+S1SVJVe2VwKdqP+1olO7pexum9SY5Lkqo6KINB0bq+VgkjW5bkHd3dzxYkeay19sBEFwVDVdV+Sa5L8vbW2o8mup4tsfSsj1prz1TVe5N8Jcm0JEtba7dV1UeSrGitLUtySQan9K7O4EyiUyeuYnY0PY7RTyTZNcnV3T7r97bW3jhhRbPD6XGcwoTqcZx+JcnxVXV7kvVJzp7s/8LI9qXHcfrHST5bVf89g8t53ukfMemnqroig0t09+r2yvpwkp2SpLX2mQzunXViktVJnkxyxsRUyo6sh3H6vzO49/BF3d9Qz7TWJu1s9/JzHgAAAIDE0jMAAAAAOoIiAAAAAJIIigAAAADoCIoAAAAASCIoAgAAAKAjKAIAAAAgiaAIAAAAgI6gCAAAAIAkyf8HjW5hZm1th3UAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pruned_model_soft = prune_soft(trained_model, percentage=0.5, min_filters_per_layer=5)\n",
    "print(\"Pruned Filters:\", 1 - count_filters(pruned_model_soft) / count_filters(trained_model))\n",
    "print(\"Architecture:\", count_filters(pruned_model_soft, cat=False))\n",
    "\n",
    "plt.figure(figsize=(20, 5))\n",
    "sns.distplot(get_gammas(pruned_model_soft, cat=True), bins=100, kde=False, label=\"pruned_soft\")\n",
    "sns.distplot(get_gammas(trained_model, cat=True), bins=100, kde=False, label=\"trained\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/160 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reducing LR at: [80, 120]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Test Acc: 71.8000%: 100%|██████████| 160/160 [49:35<00:00, 18.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final Acc: 0.7179999947547913\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "finetuned_model_soft, finetuned_logs_soft = train(pruned_model_soft, n_epochs=160, batch_size=64, bn_l1_weight=0)\n",
    "print(\"Final Acc:\", finetuned_logs_soft[\"test_acc\"][-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3deXxcdbn48c8zM9n3JmnTNm2T7i3d6cYiO1hQC4JLUQRELxcEwavXH3C9gMhFuYKiICoKCBdZZBGoyFKpFITSJW3TfW/TNl2TtNlnn+/vjzOZLE2aSZt05kyf9+vVV+csc+aZmTNPvuc53+85YoxBKaWU/TliHYBSSqneoQldKaUShCZ0pZRKEJrQlVIqQWhCV0qpBKEJXSmlEoQmdKWUShCa0JXtiMgiETkiIimxjkWpeKIJXdmKiJQAnwEMMPckvq7rZL2WUsdLE7qym2uBJcAzwHUtM0UkTUR+ISK7RKRORD4WkbTwsrNFZLGI1IrIHhG5Pjx/kYh8u802rheRj9tMGxG5RUS2AlvD834d3ka9iKwQkc+0Wd8pIv8lIttFpCG8fIiIPC4iv2j7JkTkbyLyvb74gNSpSxO6sptrgefD/z4rIgPC8x8GTgfOBPoB/w8IichQ4B3gMaAQmAKU9+D1rgBmAePD08vD2+gHvAC8IiKp4WXfB64GLgOygRuAZuBZ4GoRcQCISAFwIfBiT964Ut3RhK5sQ0TOBoYBLxtjVgDbga+FE+UNwO3GmL3GmKAxZrExxgt8HXjfGPOiMcZvjKkxxvQkof/MGHPYGOMGMMb8ObyNgDHmF0AKMCa87reB/zbGbDaW1eF1lwF1WEkcYB6wyBhz8AQ/EqXa0YSu7OQ6YIExpjo8/UJ4XgGQipXgOxrSxfxo7Wk7ISI/EJGN4bJOLZATfv3uXutZ4Jrw42uA504gJqU6pSd6lC2E6+FfAZwiciA8OwXIBQYCHmAEsLrDU/cAM7vYbBOQ3ma6qJN1IpcjDdfL78Bqaa83xoRE5AggbV5rBLCuk+38GVgnIpOBccAbXcSk1HHTFrqyiyuAIFYte0r43zjgX1h19aeBX4rIoPDJyTPC3RqfBy4Ska+IiEtE8kVkSnib5cCVIpIuIiOBb3UTQxYQAKoAl4jcg1Urb/EkcL+IjBLLJBHJBzDGVGLV358DXmsp4SjVmzShK7u4DviTMWa3MeZAyz/gN1h18juBtVhJ8zDwv4DDGLMb6yTlD8Lzy4HJ4W0+AviAg1glkee7ieE9rBOsW4BdWEcFbUsyvwReBhYA9cBTQFqb5c8CE9Fyi+ojoje4UOrkEJFzsEovJcaYUKzjUYlHW+hKnQQikgTcDjypyVz1FU3oSvUxERkH1GKdvP1VjMNRCUxLLkoplSC0ha6UUgkiZv3QCwoKTElJSaxeXimlbGnFihXVxpjCzpbFLKGXlJRQVlYWq5dXSilbEpFdXS3TkotSSiUITehKKZUgNKErpVSC0ISulFIJQhO6UkolCE3oSimVIDShK6VUgtCErmwp4PdRX1vDhiXvsmvjCoKBAMvf+A071y895vN8Xg8VG8vwNDdSfWA3h/buZPuaxZiQXi9L2Z/esUjFjWAgwJqFz+M9tB1n9UZCg07HlVWIMSHY/C6DGtbgMIZM00hIHGTTxHggZIQayWUGR6gvT2fHa/3xOdJoyBhChns/fmcaAVcG2c17yArWUkI1ISMUiHUdo/7Akb9mUe0cQM2IKygeNpLis66O6Weh1PHQhK5ixoRClM3/HYEjuyEUIOtQGVO91v2ba8kkt25BZF23SWZT5kyMOPCn5CGhIKH8kbhyB+Pfv4Gkhj3sGHIm/Ta9gM+ZjmAYVlfGYdcAMoI1pHsqqEkeTHNSHhXDvoVpOIBk5GN8zUhqNo5D6ymuXc6oLQ+zaddE0ISubEgT+iksFAzicDpj9vpl83/LjPIfRabryGDJmDsYfeF15BUMZP+erXibGwAYMHQMUzOyotjqD9pN9W/zeFA3zwz4fXzp1++SlpGltxRStqQJ/RSyr2Izu/75JKW7/0pF0WcZf+B1Nk34IY7dnzD1tr/gcDhY9srPMd4mxl52C5sXPkPG4PFMOHvuUdtqrD9CRmYO4ji+0zDNjXWMKX+ADUkTGP4f75GckkaO08nsNusMHDbmON/p8XElJZOU3R9vSC8prexJE/opomz+75i24i4GicFvnBQdsG6fOXPdfdbyR79KUcM6ZpkDADQ99gdmiQf/Biermh5h7NlXUrllFTVLX8IkpTN999PsdA7Bf9kjBLxuxs2e066131h/hKo9Wyg9bVan8az7x/8xk2a48G5S0zP7+N1HTwRCeo+AE9LsC1DV4GVIXjoOh8Q6nFOKJvRTQPn7LzJxxd1sTJlA8mU/o3bHSsav/im7kkcy3r+OOjKYXv8+W1yjWTrqBpJ3fUi6r5rQxfcj/7iXiYu/h2/xDxklXkaFt7ndVUpWsI7+b30RgCVrv05y6Rnkl0zk0MZPcG15i0lNn7LyzMeY9tlvtIvH09xI3vpn2SODGDfzkpP8aRybQwRN58fHGIM3EOL2l8r5x4aDTC7O4Tvnj2ThxoO4/SEuGtefL0waFBdJvrbZh8cfoignlT2Hm9lW1UijJ0CjN8BF4wZQmJXSbv1mX4CXlu2h2Rfg2jNLSHY6cPuC5GUkR/V6xhiW7jxMv4xkRg+IpnR4fGJ2x6Lp06cbvXxu36utPkD6Y+PZlTScwpveIregCACPuwmvu5mK8g+AEL7Vf2XCjU+R1qFO3Vh/hC1PXEswKQMZcT7DTv8se9YsYsTMz9FUV8PeN+4h1VPFRO/Ko167waThJMS+K1/nyD9/TciVhnGlMfLgOxRQy/IpP2XGFbecjI8hat94aimN3gCvf+esWIdiOw++s4mXy/ZwpNnHuaML+XR7Dd5AiKxUF5kpLvbXeTh/TCG/mjeVZxdXcP6Y/kwszqHZF8AfMOSkJ/VJXDWNXtbtq6fZG+D1VXtZtaeWZm8ATyDE5yYO5P2NB2n2BSPrjy3KYkB2KlmpLpq8AXzBEHsOu9l9uBmAof3S2V/nJhgy3H/FBCYMyuGh9zZzpNnHnNOKOGtUAS8u3U0wZBicl8aIwkz+tbWa11ZWAvD5SQO5//IJUf8x6EhEVhhjpne6TBN6Ytu1uZxhL55L2fSHmP75G/vkNZoaalm/4BnE6cKxfSGBlFwGHC5DvvIs/V68jCxxt1t/ddosmPlvTD7/y30Sz4m49ull1Lv9vHGLJvSeaPQGmP3ThTR6A7gcwid3XsChei9VjR7OGllAksPBc0t2cd/f1pOVmkSd20+yy8Gc04p4e+1+AiHDhWP781+fG4cvEGLMgKwTbskHgiEeeHsjzy6uoOW0SFaKi3PHFJKa5CTF5eBvq/dRWpDBHZeOJTctmU0H6vn+y6tJdjnITnWRn5FCVqoLp0O45fyR1DR5+fH8DcydPIjdh5v5cEsVqUkOctKSGJSbxqrdtYBVuktyOvAFQpHp754/EhHht4u2ccecsXz7M8OP630dK6FrySXBBXweAJxJKd2sefwysnKZedX3wlO3AlaXRHE4WHPeb2je/E/yZ83j8CfPEEpKY/a/PXbcJ1P7mkOsw2PVM88urqDRG+BnV06kf1YKA7JTGZCdCuRE1rnuzBKK89L47ouruHrmEOrcfv62Zh+fmziQkvwMnllcwYW/+BCAouxULhjXnxvOKiUt2cng3LR2r7e2so7+2Snh1+jcO+sO8KdPKpg3YwiXTxlMZoqLUQMySU1qPdfzP1dMQKT1D8e4gVkcafYzYVA2s4bnd7rdL04tBqw/GPfOX8+HW6p44duzGZqfTnWjl0+2VVOcl0ZJfgZBY5hfvo/RA7I4Z7R1k6HPTxpIaUFGzz7gKGlCT3BBv5XQxdV3Cb0zLQl70vlfgvO/ZM2ces5JjeF4CKCdXHrm463VPLxgMxePH8C8GUPaJciOLhw3gJV3X0yKy4GIEAiGcDmtfeWL0wbzZvk+irJT+dfWKl4p28MLS3eTmeLijVvOosHjpzArhR/PX8/7Gw8xfmA2T10/nWU7D3P6sDyK89Lbvdaq3bWkJjn4nysmRF6jo46xigjfOrs0qvftcjp44IsTMcZEtlOQmcLlUwa3W69jS3xUH9bQo0roIjIH+DXgBJ40xjzYYfkjwPnhyXSgvzEmtzcDVccn6PcCfdtCTyTWSVHN6D2xcvcRjIFfz5tyzGTeom0LuW2iHVGYyfcvHg3A12YNZcO+ej7cUsUvFmzm4kc+pOXAySHwhcmD+NvqfZz54D8xxiqlDC/MoNkX5HsXjeZzkwayurKW0wbldJnMe0s07/lk6Tahi4gTeBy4GKgElovIfGPMhpZ1jDH/0Wb97wJT+yBWdRyCkZJL14emqpUI6GVdesYXCOEQSE/u3QP+8YOyGT8oG7cvwAebq/j3c4dT1eBlwuAcTh+aB0BakoMrpxXzzCcVNPuDeAMevvviSj7eNpQVu45ww1nRtbYTRTTfwExgmzFmB4CIvARcDmzoYv2rgXt7Jzx1orSF3jOi3RZ7zBsIkuLquxHH379kDN+/5OhBZo9d3dpunB2udzf7AvzP3zfywtLdAEweknPU8xJZNAl9MLCnzXQl0OloEREZBpQC/+xi+Y3AjQBDhw7tUaDq+LQkdFeyttCjIehJ0Z7yBUIku+LjJHd6souffnEiX581lL8s38MFY/t3/6QEEs230FmBqKs9fh7wqjEm2NlCY8wfjDHTjTHTCwsLo41RnQATCCd0baFHxSGC5vOe8QXjJ6G3OG1QDj+5fAJZqX3Ttz1eRfMtVAJD2kwXA/u6WHce8OKJBqV6T6ilhZ6iLfRoOBw69L+nvP4QyX184lFFJ5pvYTkwSkRKRSQZK2nP77iSiIwB8oBPezdEdSIiLXQtuURFEE3oPeQNhkhJ0oQeD7r9FowxAazRIu8BG4GXjTHrReQnItL2MnxXAy8ZLUDGlZaEnqQJPSoiXdcTVed8AW2hx4uo+hkZY94G3u4w754O0z/uvbBUb4kk9JS0btZUEO7lohm9R3yBEClxVkM/Vem3kOBM0AdAUrKeFI2GDv3vub7utqiipwk90QWshJ6sJZeo6ND/nounbounOv0WEl3Qi884Y3qrOTvRof89F4/dFk9V+i0kOAn68HNq9cU9ESKiQ/97SE+Kxg/9FhKcBH34RBN6tERr6D3mDWi3xXih30KCk6CPgF4lOWoO7bbYY9pCjx/6LSQ4Cfnwaws9ajqwqOf0pGj80G8hwTmCPgKa0KPmcKD90HtIE3r80G8hwTlCPgJ6UrQHRLst9pA3ENJ+6HFCE3qCc4T8BBzHd3fxU5F1X2LN6NEyxmi3xTii30KCcxo/QdGTotFyiLbQe8IXtPp46tD/+KDfQoJzhnwEtYUeNRG9fG5PeANWQtdeLvFBv4UE5zJ+gnpSNGp6g4ue8YUTuvZDjw/6LSQ4Z8hPSFvoPaIt9Oj5tIUeV/RbSHAu/IScmtCj5dALovdIJKFrDT0u6LeQ4FzGT8ihJZdoaQ29Z1pq6NptMT5oQk9wScaP0ZJL1HTof89oCz2+6LeQ4JLwY7TkEjWr26Km9Gj5gkFAE3q80G8hwSWZAMapdyuKmugNLnpCuy3GF/0W4lAoGKT8/RcJ+H3HXG/1P19iyQv3H3MdbaH3jJ4U7RmvdluMK/ot9KHKbesom//7Lpcvf+M3bPj0naPmL3vpAaZ8fBPl7/7pmNsf+NFdzN7yMMte+xUAe7aupvrA7sjyUDBIsgRBE3rUrFvQaUaPRvmeWr75p+WAttDjhX4LvWztR69T/+NB1NfWkP/c+UxfeQemwy1wNi1/n+1rlzCj/EeMf28eu7eUA2BCIZa9+ktmb/0FACGf+5ivVefsB0C/Df9HKBgk9fnL2fN/N0WW+3we64FLE3q0rFvQqWj88aMdkcc69D8+6LfQy5p2l5NNEzX7dpAmVsnE3dzQbp2xf7+KEa99NjK9/71HANi88gNmrrsvMj/kPnLM10oPWdvNCNazZdUiCjnCqKaV+H1egMj/4tIaerS022L0ivPSIo/1pGh80G+hl5nmwwC462si89Y9dTP77hvNjnVL2Vz2z6Oek964C4AjaxcAsKTklnbbAggGAgQDATYtXcCujSsAyA7VA5BpGjmyar71WNxsW7XIiqGxDgBJSu+195foxKZD/wPBECV3/p3HP9gGQLMvwPf/Us7O6qbj2l6d28+v39/Kxv31Xa5T7wlEHms/9PgQVUIXkTkisllEtonInV2s8xUR2SAi60Xkhd4N0z4cnloA3If3RubNPPJ3BpmDDH/1Esa89cV2629zjiDftw+A7P2L2e4czuzrf0o1uTg9RzChEJ/+4TYa/2coGx+6kLHvfJnCly6loe4wWeImYBxkiZvCA/9iu7OUoBFq1/0DgEM71gCQMXD0yXjrCcG6fG783lf0iQ+3c8+b646av+twMwB//JdVBlm9p46/rtrLBb9YRChk8AaCvL/hIJsPNPDC0t0EgiEWbjzIw+9tpt7jp6bR22571zy5lEfe38Izn1R0GcvhptbnZKRoQo8H3V5XVUScwOPAxUAlsFxE5htjNrRZZxRwF3CWMeaIiPTvq4DjRTAQYNmffkDpnO9SNGRkZH6S10ro/t0rut1GyAjVhbOZvv9Ftpb/i1HeDaws+jIjgEZHFi5vLWsWvcwZ+57FbZKZ4C1np6OE0lAFyz94kRnAXudghoX2MCSwm9V5FxNsTCLvwCcANFZaP/yiUdN6/f0nKsHK6CEDTjm5rx0MGZzhvyhbDjbwx4928NMrJ5LU5oTjh1uq2HKwgZ9cPqHdc7ccsMpv/bOs8tqhBuv8iTGwYMNBqho83P3m+sj6CzYcwOVwsHDTQZbtPIwnEGT+rWcD0OQNsHavdXRXFU70Hn+QF5ft5sqpxVQ1evjmM8sRhJml/fjFlyeTlaqjkeNBNC30mcA2Y8wOY4wPeAm4vMM6/wY8bow5AmCMOdS7Ycafio1lnLH3GSo+fK7d/BS/ldCzD6/pdhsHpQDHgLG4JMSoNz5PrWRTfPGtADQ7c0jx1+KutH6Ee698gyWjf0jh9xbhMy5SNr0OwOHUIdbrip9gRn9qBpzFSN8mGuoO46jayBGyyO8/uLfedsKLVQt9z+FmJtz7Hp9ut0p1r66o5JUVlWw+0P78y+EmH9WNPl5Yups3VrUeBW452AhAvwzrBPj+Oiuhpyc7eXVFJcsrrPMx35g9jOvPLGHR5irKdh3GGFhWcZh1e+tw+6xBQhU1rWWafbVuqhq8fOOppdz3tw28Ub6Xsooj7DnsZvfhZgozUxjST0t68SKahD4Y2NNmujI8r63RwGgR+URElojInM42JCI3ikiZiJRVVVUdX8QxVLGxjLoj1QDU7rESrRze0W6d9IBVcyzxbul2ezUpg8ksGhWZPnzpExSPtFpe3qQc0gP1SP1e6kln5OSzmP21/yYzO4+tKacxyWN1F/Nml0aeL5n9yR5/MS4JsX3Zu2Q3bGdfcgni0FMl0ZJwQj/Zg4t2H27G7Q/y6MKtAKzYZSXgjgm9psk60f5fr6/le38p50h4estBaz2P3+pRdaDOQ1aKi2tmD2PR5kO8t/4An5s4kPuvmMClE4oAqG32R7YbMrDpgLXvttTdZ5b2Y3tVI5f++l+s3G01VLYeamB3uLwDkJehLfN4Es0vvbMDz467uwsYBZwHXA08KSK5Rz3JmD8YY6YbY6YXFhb2NNaYWvK7myj5y4Vs+vMPAPAdtBJ2RtPudutlhE9Upkv7mmRHa1Jn4Jn4dQqGjo3MGzPjoshjf0oumaF6kpv3U+No/1nVDzwj8ljyR0QeJ2UPYOTp5+M2yXi2fsBg304as0ehoifhjG56qfNindsfSbotfvbORt5Zu7/dvCavdYLx0x01LN1Rw9pKq+Sx5WAD/mCIQw0ejDFHbeuZxRUAbA4n9NpmHx5/kAN1HopyUpk3YwiBkMEbCDF1qPWTHDcoO/J8h0BSuLa0bp+171aEE/qZI/LxBw3VjV4eu3oq04flseVAI3uOtHan7ZehPajiSTQJvRIY0ma6GNjXyTpvGmP8xpidwGasBJ8QGuoOM/3AXwAoOlIGQNKR7QAUeFsPe00oRI5pbVHVkRF5vKT/V1l/yYuR6Zwrfs70z99IQdHQyPK2Lelgah7ZppFs7wHqUwa0iydzxOzI44xBrSc8U3MHkpKazq7kkZQeWkiWuGFA+1qrOjaJlFx6Z3t3vLqGm/7cej7FGwjyxIc7uPn5lZHyCkBzuNzhdAjXPLU0cmu3dfvquPAXHzLzgYVUNXoJdDh0WLjpIK+vqmTbIavkUlHTzNi73+Xd9QcoyklleGEmV00rBmB6iTVuITs1KdLl8P4rJvDEN04nLz2J9eG6+Y7qJoqyUxnZPzPyOrNK+zG6KItNB+rZ3aYkk5+hYxziSTQJfTkwSkRKRSQZmAfM77DOG8D5ACJSgFWC2UGC2L78PVwSYkPSBAYH99HcWEd2s9XVsL+pxuO2dnB3cwMp0noYuzeptRxiUrLJzB8Umc7Ms84bO5xOQncfZtZN7UeUSlo/UsTPoEAlnvSB7ZYNnfiZyOPCYeNbt5lvrVeXO54BWMmi/2nnHP8bPwU5WlrovZTQNx6oZ8P++khNfm+b1u1fV1ZGHjf5rBb6V6YX4w8a8jOSOXtkAZ9sq4mUON5dd6Ddtj83cSCb9jdwz5vrmVnaj1vOH9FueVF2KgD/e9VEXv73M5gypPWgefxAq5X+hcmDuGDsAMYNzGZTuLxTUd1ESUE6g3Jb+5nnZ6YwtiiLek+A1eGjB2it2av40G1CN8YEgFuB94CNwMvGmPUi8hMRmRte7T2gRkQ2AB8APzTG1HS+RXv59KkfMOVja/Sl5/R/wyUhKtYuZqB/D0fIwiGGgxWbAKg/0v68QEN26w9MUrPJLWw99ZDTr7UjkMPpPKrO7cjIByBNfISyBrVblpNXEHmcP6D14Kll+47BU63XN2kMG6M9XHrCEamhR5/RGzx+/ra640Gr1Td87xE3DZ4Ah8OlkpZyRUayk1V7avnR62tZt7eOZq/VQv/+xWP436sm8s8fnMfF460js/83ZwwpLgevrWj9A5DicjB3yiACIUODJ8DN546gILN9+SM33apvu5wOZpb2a7fsiqmDuWLKILLDvVMG5qRxsN6DPxhi84EGRg/IYnCbhA4wZkBW5HFWitVBThN6fInqdvDGmLeBtzvMu6fNYwN8P/wvYZTN/x1n7HkSgHUpUxg65UJYAuPfmwcCSwuuYlb1axxY8w+GjTud+qq9FAEek0Sq+DE5w/BUW48dqdlk5+bjM048kkJ20rF/CElZrUnblTf0qOXrkydymm8trqRkGk0ayfjIzrNq7QWjZkA57Ew7jUlO7R/cE63dFqNP6H9duZd7569ncnEuQ/Nbe3zsr/NESiSPLtzK5CG5NIVLK5dNHMgrK6xSyYdbqphzmnWisl9GMl+dYX3f18wexgVj+zOkXzrLdh5m0WarwVCQmUxxXjqnD8sDIDPFxZkj86lztx4dwrGPMi6bOJDLJrYe+RXlpHCowcuq3bU0+YKcOSKfwswUBuWkcusFVvX09GF55KQlUef285nRBby99gADc1Kj/pxU34sqoZ+q8st/zzbnCJj7GP0LBlFQNIQlA64mubGS0Li5TLvkOjY8vIWpG37O0t9sZlb1XwE44BxISWg3rvyhNO9IIxU/rvQcxOGgVnLwSxLZ3bz2gFHTYbH1OL1g2FHLR//nQjzBAKlAg2TiJEj/cCt/yOipHCQfd8lFRz1PHVukht6D5+yttVrdlbXN7RJ6294gz366Cz61ynTJLgcXjhvAKysqcTmEyiNuXl+1l7QkZ6QfOlj19JYugWePLIgk9D9cO50B2akUhMsgEwfnkOJyRlrkAPd8fnykdh6NouxUgiHDm+V7EYHZw/NxOITFd10YWcfldPD3287mwXc2cc8XxvNvnxnO8MLMY2xVnWya0IENS97F/cnvmXL7qzhd1kdSV3OQ0lAFn5bczBmTz4qsO/vm9rXu4u+8ye7HLo4kc4DD6aWUNO4ms38pbkkDU09ShlW/rHfmEZLuW82DSsdS8dWF7F/8IqdPv/Co5UnJKSRhHWI3O7MwCC1FHFdSMv3v2UbCj+7qA5FeLqFjr9fkDfDowq3cftEo9oUT+r5aT7t12iZ0sHqT+IOG/lkpTB6SA8CcCUW8tWY/NU0+CjK7Pmo7Y0R+5PG4omzSkq196JWbzogMPMpLb33+DWeX0hMDwvX2v67cy8TBOeSmdx5LcV46v/maVcbrn6Wt83ijCR1oKPsLsxo+YMuajxk97Tw2lS2k6cPHOB3IGXveMZ+bnZuP/8b5fPrWrxgx5xaqdm3EV7ECtnxI/uCR1DvSIQgpmdbh8ZGJN0QdV8m46ZSMm97telVDLsU4nIxsM0/7nh+fyMCibtroLyzdzRMf7SAzxcWB8CCelsQOsGD9Ae7669p2z3niG6dzwzNlVB5xMzAnjV99dQpnjMjnnXUHCIYM6cld/xzHFbUe07Ukc6DdCM2WhN62lR+tonDpxO0PMqOkXzdrq3ilCR3IqrcGc9SseRcz5RxK/vZVUsO9VYZP/syxngpA/oBizvjWwwD0H1xK3ZjTWdlvCNMGlVDtSLMSeriFPuOKW3s9/tnX/7TXt3mqakmF3Q0sakmaBxs8kVGZ++taE/rDCzYDMDg3jevPLEEEzhvdn2Sng+vPKgGsE5MAuWlJ1DT5SE/u+sjN4RBSXI7IDSU60zLI5/wxPT82a+kRAzCpOKfHz1fx4ZRP6CYUYrDP6mGZve9jNi59j/HiJ2AcrCiYy6y0jG62cLSc/AFMm3M9AH5nBvghIyuvN8NWfcThaOm2GF0V/UiTnwP1VkLfGy651Db72HqokSunDua7F46itKB1H9p0/5zIa7TISbcSekbKsX+Oi++8gIY2VzjsKCs1iTduOYvRA3pe185v00NmcvFRYwKVTZzyCb36wG4KaaKWTEZ5N7B28R9oNGk4friZWZkn3lIJuKyTWhk5ehhrBy019O5a6C09SrYcbCAYXrml5LJ4e+LGy7gAAB32SURBVA3GwNdmDW2XzIGjkjlYLXTgmC10sJJufuaxR2a27WveE23LNMPy9dosdnXKF1r3b7FG8W0c/CWSJcjU+g/YkjWD9F5I5gABVwYB4yD1OFr66uRrSWvdtdBbEvrW8AjN0oIM9tW6OdTg4dGFW8lOdTE5yuSaE07oGceooZ8MGclO+mUkR/6oKfs55RN6c6V14mr4pbfhNUk4xBAovaDXth/MG06lc7CepLSJyEjRbtarbW5/TZVZpf1o9gW567W17Kxu4vGvT2t32dtjaelRkh7ja4ov+9FFfHzH+TGNQZ2YUz7LOKs2Uk0uA4pHsCXVuu7JkOmX9dr2Z13zEwbdUdZr21N9S6IcKVrbZhDP4Nw0rjuzBICFmw5x3phCPjMq+ovPxU0LPcV1zJ42Kv6d8gk9t3Eb+1OGAxCceRNLC65i4LAxvbZ9h9NJcor217ULR5QX56pt9jO8IIOLxw/glZvOYGxRFoPCXf/OHd2zXiYtA4Ji3UJX9ndKJ/RgIMCQwC6acq0rFk65cB6zbn06xlGpWIp26H+928/4Qdn88drpDMpNQ0Q4b6yVyM8ZXXDM53aUGyctdGV/p+Qe5PN6SE5JZV/FRoaIH0fRabEOScWJaC+fW+v2txtqD/DdC0Yye3g+xXk96yUSqaF308tFqe6cci30VQv+TPLPBrB97RIObbFq27nDJsc4KhUvorl8bihkqG32kZvWfnj8wJw05k4e1MWzupYT/sPQXT90pbpzSu1BPq+H0z65DQSqN35E2o53qSaXYeNnxDo0FSeiOSna6AsQMhzVQj9e0fZDV6o7p1QLfcPHb5As1uVLU3f+g0meFWwt/TopqTqQQlm667YYChke/2AbANlpvZPQxxRl8bmJA/UaKuqEnVItdO/G92g2KdQ6cpjsXgbAqM/eHOOoVDzproW+urKWJz60LhXRW7dfS0928fjX9UYk6sSdEgm9qaGWisc+zyzfWlanz8IgDHIfolKKKC4a0v0G1ClDuqmh7wzfQPnfzx3OOaPtdaNzlfhOiZLL1mXvcprPGhHqKbkQT5Z1w4gDWXoDZdVed0P/K6qbcAj84OIxUY8EVepkOSX2SM/mf+IzLtac+xSnX/kfSH54INGg7q81rk4t3dXQd9Y0MzgvjWTXKfHTUTaTsHtlc2Mdfp8XgAHVS9mSOoFJ538JV1IyeaPOwGecFE2+JMZRqnjT3U2id1Y3Ulqgt11T8Slha+iHfnk2+wd/FmfDXmaGKlgy5HuRZaOnnYtn7G6GpesPU7UXOSnayX0kjDFUVDdz+lC9tr2KTwmb0PsHD3KkZgNTmz+hLPsipn35rnbLUzWZq05ETop2UnSpavTS6A1QUqCXQlbxKSFLLqFgkHTxkuvZa00PPUsvkKWi0npS9Ohlm/Y3ADBmQNbJC0ipHogqoYvIHBHZLCLbROTOTpZfLyJVIlIe/vft3g81el6Pdbf1AcH9ALgy9BBZRedYQ//X76sHYPyg7KMXKhUHui25iIgTeBy4GKgElovIfGPMhg6r/sUY0/t3QD4O7qZ60oB0sU6KJmflxzYgZRvHGli0YX89g3PTIhfTUireRNNCnwlsM8bsMMb4gJeAy/s2rBPjaW5sN52qCV1F6VjdFtfvq+M0bZ2rOBZNQh8M7GkzXRme19FVIrJGRF4VkU6HX4rIjSJSJiJlVVVVxxFudHzuhnbTGbk9uz61OnV11UJv8gbYWd2k5RYV16JJ6J3dMbZjA+ZvQIkxZhLwPvBsZxsyxvzBGDPdGDO9sLDvhk373E3tpjNzdYi2ik7r0P/2u/iKXUcwBk4fpudjVPyKJqFXAm1b3MXAvrYrGGNqjDHe8OQfgdN7J7zj43e3llwCxkFmVnR3X1eqq1vQLdlRg8shmtBVXIsmoS8HRolIqYgkA/OA+W1XEJGBbSbnAht7L8SeC3hbSy4Nkok4ErJ3puoDrbegaz9/yY4aJhXn6E2UVVzrNtMZYwLArcB7WIn6ZWPMehH5iYjMDa92m4isF5HVwG3A9X0VcDQCHnfkcaPoACIVvdYWemtGP9TgYU1lHbOH68l1Fd+iam4YY94G3u4w7542j+8C7ur4vFgJeltLLs1OHQSieiByUrR11hMf7sAAX5mul1pW8S0haxHG13pS1OvSXgkqeo4OQ/+NMby4bDdfmDRQh/yruJdwCf3Q3p2EavdGpn3JOTGMRtlNx5Gitc1+mn1BJhbriXUV/xLuDE//P06hf5vpYLK20FX0OvZDr260Om8VZqXEKiSlopZwCb2F2ySzpuAy0iZ8PtahKBvp2G2xKpzQCzJ1uL+Kfwmb0D2Syqzvdjq+SaljaOm22NJC9wFQmKktdBX/EqqGHgoGI4/9ifu3SvWhSAs9PF3d0NJC14Su4l9CJfTGhtrI4zTjPsaaSnWu49D/6kYvLoeQk5YUy7CUikpCJfTmhiORx1miCV31XMcaenWjl/zMZByOzi5ppFR8SaiE7m6T0JU6Hi3dFkORhO7TcouyjYRK6N7G2u5XUioKbbstakJXdpFQCd3XVBfrEJTNdRxYVNWgCV3ZR0IldH9zawt9bcrUGEai7Eo6XJyrttlPXrqeEFX2kFB9+4Ju6ya+e69dzPihY2IcjbKjtregC4UMbn+QjJSE+pmoBJZQLfSQxyq5ZOcPxOnSH6HqubZD/91+a1xDRoozhhEpFb2ESuh4GggZISNTL8iljk/bbotNvgCA3tRC2UZiJXRvPU2k4nBqi0odH5HWof/NXquFnp6s+5Oyh4RK6E5/I02i16xWx69l+JC20JUdJVRCd/kb8DjSYx2GsrG2N7hw+7SGruwloRJ6UqARj1PvIaqOX+SkaAiafC0lF22hK3tIqISe4a/Fo7ecUyegbbfFZq9VctEWurKLhEnooWCQouA+PNmlsQ5FJYCQMa0t9CRtoSt7SJiEfrByG2niQwpGxToUZWOONhdEb245KaotdGUTCZPQq3auAyCreHyMI1F25mgzsKgp3G0xQ2voyiYSJqE3798EQP/SCTGORNmZ0Hr5XLcvgAikJiXMz0QluKj2VBGZIyKbRWSbiNx5jPW+JCJGRKb3XojRkZpt1JNOfv/BJ/ulVQJprbhYNfT0JGdksJFS8a7bhC4iTuBx4FJgPHC1iBxV1xCRLOA2YGlvB9mdYCBA/8Mr2Ocahji0NaVOQKTkYtXQ0/XCXMpGosl+M4Ftxpgdxhgf8BJweSfr3Q/8HPD0YnxRWfG331Ea2kXjlBtO9kurBONovX4uTd4gGTrsX9lINAl9MLCnzXRleF6EiEwFhhhj3jrWhkTkRhEpE5GyqqqqHgfblbRNr1HhGMLpl32717apTk0txZVIC11PiCobiSahd1ZANJGFIg7gEeAH3W3IGPMHY8x0Y8z0wsLC6KPsRoG3kurMMVpuUSes9Y5FhmZfUAcVKVuJJgNWAkPaTBcD+9pMZwETgEUiUgHMBuafrBOjHncTA0w1/hwdUKROXNubRDf5gqRpC13ZSDQJfTkwSkRKRSQZmAfMb1lojKkzxhQYY0qMMSXAEmCuMaasTyLu4OCuzTjEkFQ48mS8nEp0bfqhN3sDWkNXttJtQjfGBIBbgfeAjcDLxpj1IvITEZnb1wF250jlZgCyBo2OcSQqETjaFBibvAHSNKErG4nqeNIY8zbwdod593Sx7nknHlb0PAe3ADCgREeIqhPX0ufcHzQcavBSlJ0a44iUip7tzyLK4R3UkUFuQVGsQ1EJoKWFvq/WTSBkGNpPr6+v7MP2CT3Je5haR16sw1AJomXof0VNEwBD8zWhK/uwfUJ3Btz4RA+LVe9oGVe0q6YZQFvoylZsn9BdIQ9+Z1qsw1AJoqXb4u7DzSQ5hYE5um8p+7B9Qk8OeQg4tIWuekfb63AV56XjdOiFuZR92D6hJ4U8BLSFrnqJo01GL87T/UrZi+0TekrIQ9ClLXTVO9q2x/MzkmMWh1LHw/4JHS8hl564Ur2jbckl2WX7n4c6xdh+j001XoxLD41V72h7MwtN6MpubL3HmlCIdPFCkrbQVe9pOQ+a7NRh/8pebJ3QPW5r8IdJzohxJCqRtJwY1Ra6shtb77HupnoAJElLLqr3aUJXdmPrPdbrbgTAmaItdNV7AiHr/i0pmtCVzdh6j/U1WwndoQld9QFN6MpubL3H+jxWDV1b6KovaMlF2Y2t91hfuOTiStWErnpfstPWPw91CrL1HhvwWi10TeiqL2gLXdmNrffYoMdqoaekZcY4EpWINKEru7H1Hhv0WtesTtaErvqAllyU3dh6jw35rJJLSnpWjCNRiUhb6MpubL3HmnBCT9WErvqAJnRlN7beY43PDUCaJnTVB7QfurIbe++x/ia8JgmnyxXrSFQCSnHpxbmUvUSV0EVkjohsFpFtInJnJ8tvEpG1IlIuIh+LyPjeD/VoDr8bj+hNCFTf0JKLsptu91gRcQKPA5cC44GrO0nYLxhjJhpjpgA/B37Z65F2wumtpVG0h4vqG9rLRdlNNHvsTGCbMWaHMcYHvARc3nYFY0x9m8kMwPReiF1L9xykNqn/yXgpdQrSFrqym2j22MHAnjbTleF57YjILSKyHauFfltnGxKRG0WkTETKqqqqjifednIDVbhTB5zwdpTqjCZ0ZTfR7LHSybyjWuDGmMeNMSOAO4D/7mxDxpg/GGOmG2OmFxYW9izSDkLBIAWhGvwZg05oO0p1RRO6spto9thKYEib6WJg3zHWfwm44kSCisaR6v0kSwBHjiZ01Te0hq7sJpo9djkwSkRKRSQZmAfMb7uCiIxqM/k5YGvvhdi5IwcqAEjuN+TYKyp1nDShK7vptgO3MSYgIrcC7wFO4GljzHoR+QlQZoyZD9wqIhcBfuAIcF1fBg3QeGg3AJmFQ/v6pdQpyuHorNqoVPyKakSOMeZt4O0O8+5p8/j2Xo6rW97D1nnavIElJ/ullVIqLtn2mDJUtxe/cdKv8KgON0opdUqybUJ3eOtolAwcTh2erZRSYOeEHnDjltRYh6GUUnHDtgndGWjGK2mxDkMppeKGbRO6K9iMz6EtdKWUamHbhJ4cdON3agtdKaVa2DahJ4Xc+J3psQ5DKaXihm0TekrITdClCV0ppVrY9lY/qcajCV31idMGZTMgW8/PKPuxbUJPN25MUkasw1AJ6O+3fSbWISh1XGxZcgkFg6SLF5OsCV0ppVrYMqF73I0AiCZ0pZSKsGVCb2607ninCV0ppVrZMqF7m62E7kjRG0QrpVQLWyZ0T1MDAK40TehKKdXClgnd7w4n9NSsGEeilFLxw9YJPSlVW+hKKdXClgk94Akn9PTsGEeilFLxw54JPdxtMTVDSy5KKdXClgk95GsCIEVb6EopFWHLof/Ga7XQ07SFrlSP+P1+Kisr8Xg8sQ5FdSM1NZXi4mKSkpKifo49E3q4hZ6WrgldqZ6orKwkKyuLkpISRCTW4aguGGOoqamhsrKS0tLSqJ9ny5KL+N14TJLeIFqpHvJ4POTn52syj3MiQn5+fo+PpOyZ0AMevJIc6zCUsiVN5vZwPN9TVAldROaIyGYR2SYid3ay/PsiskFE1ojIQhEZ1uNIekCCHryk9OVLKKWU7XSb0EXECTwOXAqMB64WkfEdVlsFTDfGTAJeBX7e24G25Qh68WkLXSml2ommhT4T2GaM2WGM8QEvAZe3XcEY84Expjk8uQQo7t0w23MGPfhFW+hK2dGjjz7KuHHjyMvL48EHHzyubdTW1vLb3/62lyPr3HnnnUdZWVmvbGvTpk1MmTKFqVOnsmLFil5/D9H0chkM7GkzXQnMOsb63wLe6WyBiNwI3AgwdOjQKEM8mjPoxe/QhK7Uibjvb+vZsK++V7c5flA2937htGOu89vf/pZ33nmnR703OmpJ6N/5zneOexux8MYbb3D55Zdz3333UVFR0evvIZoWemeVedPpiiLXANOBhzpbboz5gzFmujFmemFhYfRRduAMeQhoQlfKdm666SZ27NjB3LlzeeSRR7j11lsBuP7667nttts488wzGT58OK+++mrkOQ899BAzZsxg0qRJ3HvvvQDceeedbN++nSlTpvDDH/6QRYsW8fnPfz7ynFtvvZVnnnkGgJKSEu69916mTZvGxIkT2bRpEwBNTU3ccMMNzJgxg6lTp/Lmm28C4Ha7mTdvHpMmTeKrX/0qbre7y/cTDAa5/vrrmTBhAhMnTuSRRx4BoLy8nNmzZzNp0iS++MUvcuTIEd5++21+9atf8eSTT3L++ecf9R56QzQt9EpgSJvpYmBfx5VE5CLgR8C5xhhvr0TXhaSQF49LR4kqdSK6a0n3hd///ve8++67fPDBB7z11lvtlu3fv5+PP/6YTZs2MXfuXL70pS+xYMECtm7dyrJlyzDGMHfuXD766CMefPBB1q1bR3l5OQCLFi065usWFBSwcuVKfvvb3/Lwww/z5JNP8sADD3DBBRfw9NNPU1tby8yZM7nooot44oknSE9PZ82aNaxZs4Zp06Z1ud3y8nL27t3LunXrAOvIAeDaa6/lscce49xzz+Wee+7hvvvu41e/+hU33XQTmZmZ/Od//icVFRXt3kNviKaFvhwYJSKlIpIMzAPmt11BRKYCTwBzjTGHei26LiSFfAQdelJUqURyxRVX4HA4GD9+PAcPHgRgwYIFLFiwgKlTpzJt2jQ2bdrE1q1be7ztK6+8EoDTTz+dioqKyLYffPBBpkyZwnnnnYfH42H37t189NFHXHPNNQBMmjSJSZMmdbnd4cOHs2PHDr773e/y7rvvkp2dTV1dHbW1tZx77rkAXHfddXz00Uc9jvl4dNtCN8YERORW4D3ACTxtjFkvIj8Byowx87FKLJnAK+G+k7uNMXP7Kugk4yXkTO2rzSulYiAlpbWMaoyJ/H/XXXfx7//+7+3WbUnKLVwuF6FQKDLdcUBOy7adTieBQCCy7ddee40xY8YcFUu0fcDz8vJYvXo17733Ho8//jgvv/xypOwSC1H1QzfGvG2MGW2MGWGMeSA8755wMscYc5ExZoAxZkr4X58lc4Bk4yWoCV2phPfZz36Wp59+msZG6/pNe/fu5dChQ2RlZdHQ0BBZb9iwYWzYsAGv10tdXR0LFy6MatuPPfZY5I/HqlWrADjnnHN4/vnnAVi3bh1r1qzpchvV1dWEQiGuuuoq7r//flauXElOTg55eXn861//AuC5556LtNbb6vgeeoMtr+WSjA/j0oSuVKK75JJL2LhxI2eccQYAmZmZ/PnPf2bEiBGcddZZTJgwgUsvvZSHHnqIr3zlK0yaNIlRo0YxderUbrd99913873vfY9JkyZhjKGkpIS33nqLm2++mW9+85tMmjSJKVOmMHPmzC63sXfvXr75zW9Gjg5+9rOfAfDss89y00030dzczPDhw/nTn/501HPz8/OPeg8nSlr+Op1s06dPN8fbt9N9byGri65i9s2/7+WolEpsGzduZNy4cbEOQ0Wps+9LRFYYY6Z3tr7truViQiHSxIdJSot1KEopFVdsV3Lxet2kAuLShK6UOnlmzZqF19u+R/Zzzz3HxIkTYxTR0eyX0D1WQidJa+hKqZNn6dKlsQ6hW7Yrufjd1s0tREsuSinVju0SutdjXQPMkawJXSml2rJdQvd7rRa6JnSllGrPfgk93EJ3JqfHOBKlVE8d72VvL7vsssh1Uk5UZmZmr2wnHtkvoXs1oStlV10l9GAweMznvf322+Tm5vZVWAnDdr1cguGEnpSiJRelTsg7d8KBtb27zaKJcGnXN61oe8nYpKQkMjMzGThwIOXl5WzYsIErrriCPXv24PF4uP3227nxxhsB6xK4ZWVlNDY2cumll3L22WezePFiBg8ezJtvvklaWhrbt2/nlltuoaqqivT0dP74xz8yduxYdu7cyde+9jUCgQBz5szp3fcbZ2zXQg+EE7orVVvoStnNgw8+yIgRIygvL+ehhx5i2bJlPPDAA2zYsAGAp59+mhUrVlBWVsajjz5KTU3NUdvYunUrt9xyC+vXryc3N5fXXnsNgBtvvJHHHnuMFStW8PDDD0duHHH77bdz8803s3z5coqKik7em40B+7XQ/dbF5pNSNKErdUKO0ZI+WWbOnNnuzkWPPvoor7/+OgB79uxh69at5Ofnt3tOaWkpU6ZMAVovh9vY2MjixYv58pe/HFmvZRDQJ598Ekn63/jGN7jjjjv69D3Fku0SesinCV2pRJGRkRF5vGjRIt5//30+/fRT0tPTI9co76jtZXadTidut5tQKERubm6XN4uI9nK4dme7kosJJ/TkNE3oStnNsS4ZW1dXR15eHunp6WzatIklS5ZEvd3s7GxKS0t55ZVXAOta56tXrwbgrLPO4qWXXgKIXBY3UdkvoYdLLsmpGd2sqZSKN20vGdvxPppz5swhEAgwadIk7r77bmbPnt2jbT///PM89dRTTJ48mdNOOy1yj9Bf//rXPP7448yYMYO6urpeey/xyHaXz1214M+Y1X9h4u2vkpSsN4pWqif08rn20tPL59quhj71kmvgkmtiHYZSSsUd25VclFJKdU4TulKnmFiVWVXPHM/3pAldqVNIamoqNTU1mtTjnDGGmpoaUlN7dt8H29XQlVLHr7i4mMrKSqqqqmIdiupGamoqxcXFPXqOJnSlTiFJSUntRmaqxKIlF6WUShCa0JVSKkFoQldKqQQRs5GiIlIF7DrOpxcA1b0Yzsmm8ceOnWMHjT/W4iH+YcaYws4WxCyhnwgRKetq6KsdaPyxY+fYQeOPtXiPX0suSimVIDShK6VUgrBrQv9DrAM4QRp/7Ng5dtD4Yy2u47dlDV0ppdTR7NpCV0op1YEmdKWUShC2S+giMkdENovINhG5M9bxdEdEKkRkrYiUi0hZeF4/EfmHiGwN/58X6zhbiMjTInJIRNa1mddpvGJ5NPxdrBGRabGLPBJrZ/H/WET2hr+DchG5rM2yu8LxbxaRz8Ym6kgsQ0TkAxHZKCLrReT28HxbfP7HiN8un3+qiCwTkdXh+O8Lzy8VkaXhz/8vIpIcnp8Snt4WXl4Sy/gB6zKNdvkHOIHtwHAgGVgNjI91XN3EXAEUdJj3c+DO8OM7gf+NdZxtYjsHmAas6y5e4DLgHUCA2cDSOI3/x8B/drLu+PA+lAKUhvctZwxjHwhMCz/OAraEY7TF53+M+O3y+QuQGX6cBCwNf64vA/PC838P3Bx+/B3g9+HH84C/xPLzN8bYroU+E9hmjNlhjPEBLwGXxzim43E58Gz48bPAFTGMpR1jzEfA4Q6zu4r3cuD/jGUJkCsiA09OpJ3rIv6uXA68ZIzxGmN2Atuw9rGYMMbsN8asDD9uADYCg7HJ53+M+LsSb5+/McY0hieTwv8McAHwanh+x8+/5Xt5FbhQROQkhdspuyX0wcCeNtOVHHuHiQcGWCAiK0TkxvC8AcaY/WD9CID+MYsuOl3Fa6fv49ZwWeLpNiWuuI0/fPg+FauVaLvPv0P8YJPPX0ScIlIOHAL+gXXUUGuMCYRXaRtjJP7w8jog/+RG3J7dEnpnf/3ivd/lWcaYacClwC0ick6sA+pFdvk+fgeMAKYA+4FfhOfHZfwikgm8BnzPGFN/rFU7mReP8dvm8zfGBI0xU4BirKOFcZ2tFv4/7uK3W0KvBIa0mS4G9sUolqgYY/aF/z8EvI61kxxsOTQO/38odhFGpat4bfF9GGMOhn+oIeCPtB7Wx138IpKElQyfN8b8NTzbNp9/Z/Hb6fNvYYypBRZh1dBzRaTlZkBtY4zEH16eQ/Tlvj5ht4S+HBgVPuucjHUiYn6MY+qSiGSISFbLY+ASYB1WzNeFV7sOeDM2EUatq3jnA9eGe1vMBupaSgPxpENd+YtY3wFY8c8L91YoBUYBy052fC3C9dengI3GmF+2WWSLz7+r+G30+ReKSG74cRpwEdZ5gA+AL4VX6/j5t3wvXwL+acJnSGMm1mdle/oP68z+Fqza1o9iHU83sQ7HOou/GljfEi9WnW0hsDX8f79Yx9om5hexDov9WC2Qb3UVL9Yh5+Ph72ItMD1O438uHN8arB/hwDbr/ygc/2bg0hjHfjbWIfsaoDz87zK7fP7HiN8un/8kYFU4znXAPeH5w7H+0GwDXgFSwvNTw9PbwsuHxzJ+Y4wO/VdKqURht5KLUkqpLmhCV0qpBKEJXSmlEoQmdKWUShCa0JVSKkFoQldKqQShCV0ppRLE/wchroOT/zKEjgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title(\"Accuracy\")\n",
    "plt.plot(trained_logs[\"test_acc\"] + finetuned_logs_soft[\"test_acc\"], label=\"finetuned_soft\")\n",
    "plt.plot(trained_logs[\"test_acc\"], label=\"trained\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dfZRU1Znv8e9T3dANtAoCyRjR0Cq+YCSILXGuudHRqCB3wIxoTG4iomsRTUxMMs4EV5aOmsyKRmdMNHiNRrIcY2J8SUaS4EsSw5hJCLFVRF50QEKk0SiCEEHemnruH3Wq+oVuep/qwnOqzu+zFlB16lTx9KlTT+969j57m7sjIiK1K5d0ACIism8p0YuI1DglehGRGqdELyJS45ToRURqnBK9iEiNU6IXEalxSvQi3ZjZGjP7aLdtp5pZ3sy2mNnbZvaSmc1MKkaROJToRcK96u5NwP7Al4C7zOyohGMS6ZMSvUhMXjAf2AiMSzoekb7UJx2ASLUxsxzwf4ARwKqEwxHpkxK9SLj3mdkmYBCFz86X3f25hGMS6ZNKNyLhXnX3oRRq9LcCpyUcj0gQJXqRmNx9B/AV4DgzOyfpeET6okQv0rMBZtZY/EO3Mqe77wT+DbgmkehEYjDNRy/SlZmtAd7fbfPvgNHuPqrTfoOBV4CZ7v6zdy9CkXiU6EVEapxKNyIiNU6JXkSkxinRi4jUOCV6EZEal7orY0eMGOGjR49OOgwRkaryzDPPvOnuI3t6LHWJfvTo0bS2tiYdhohIVTGzP/f2mEo3IiI1ToleRKTGKdGLiNQ4JXoRkRqnRC8iUuOCEr2ZTYoWQ15lZrN7ePwjZvasmbWb2fRuj80ws5XRnxmVClxERML0mejNrA6YA0wGxgKfMLOx3XZ7BbgI+GG35x4I/AvwIWAi8C9mNqz/YYuISKiQcfQTgVXuvhrAzO4HpgHLizu4+5rosXy3554F/NLdN0aP/xKYBPyo35F388a6P7H60dsq/bI168AJUzlywqlJh1FV/rxhKz95dh2a8bVvA+tzfOqk9zN08MCkQxHCEv3BwNpO99sotNBD9PTcg7vvZGazgFkAhx56aOBLd7Xp9VeYuHZuWc/Nmpw5z294AZToY/nholf47lOrMUs6knQr/h486IBBnHvCqL3vLO+KkETf02kd2qQJeq673wncCdDS0lJWc+nICafAhE3lPDVzXvr6RMx3Jx1G1WnPO00N9Sy97qykQ0m1trfe4cM3/obdeX3zSYuQztg24JBO90cBrwa+fn+eK/uMYSo/xJZ377HlIl1Z9JXHg9uDsq+FJPqngTFm1mxmA4ELgHmBr/84cKaZDYs6Yc+MtkmC3HKEfymTInd6/o4qXeSiY6S2RHr0mejdvR24nEKCXgE84O7LzOx6M5sKYGYnmlkbcB7wXTNbFj13I/A1Cr8sngauL3bMSnIK+UqfwnIoz/fNoqOkyk16BM1e6e7zgfndtl3T6fbTFMoyPT13LqBe0lRR6aYc7k4up1Tfl2JntUo36aErYzNIpZvy5F0t+hCm0k3qKNFnkEo35XG81NEovSuWbnS9QXoo0WeSqblVBleLPkhH6UbSQok+g9xyatGXwUEt+gC54vBKnWKpoUSfUUr08bm7rooNUDxEeWX61FCizyBX6aYsKt2EUWds+ijRZ5BKN+VxRy36AB1XxkpaKNFnkmF0n2hU+uJ4qf4sveto0SvVp4USfQZpxpbyaBx9GHXGpo8SfRaZYa4WfVyF0o1SfV/UGZs+SvSZZKrRl0GX9IfROPr0UaLPoMIUCBKbQ06Hrk8q3aSPTtsMcnXGlqUwH71KN6FUukkPJfqM0uyV8RWujE06ivTTMUofJfosspzapWXQBVNhOko3akykhRJ9Bql0Ux4HjaMP0DHqJtEwpBMl+kwyNCYiPleTPoipMzZ1lOizyNSlWA7l+TClNWPVmEgNJfoMUummPFp4JEzxGKl0kx5K9FlkWjO2HGrRx6RzLDWU6DPI9baXxV2dsaFypl6gNNEnPotMpZty5LXwSDAz0wVTKaJEn0ma66YcOmLhclrbJlWU6DPITYm+HJq9Mpxh6oxNESX6TFJnbHm8NHRQ+mAaXpkmSvRZpCkQypLXUoLBcromL1WU6DNI4+jL45q9MlihdKNMnxZK9BmlGn18mr0ynKkzNlWU6LNIC4+URZ2x4XKmpkSa6BOfRWbkVLqJrbDwiIQwtPBImijRZ5DSVfnUoA+k0k2qBCV6M5tkZi+Z2Sozm93D4w1m9uPo8UVmNjraPsDM7jGzF8xshZldVdnwpSyWU42+DJrrJpymikiXPhO9mdUBc4DJwFjgE2Y2tttulwBvufsRwC3AjdH284AGdz8OOAH4TPGXgCQrp0Qfm+NKYIHMVLpJk5AW/URglbuvdvedwP3AtG77TAPuiW4/BJxuhV4rB4aYWT0wCNgJ/LUikUv5LIcGOceXz6t0E8pQ6SZNQhL9wcDaTvfbom097uPu7cBmYDiFpL8VeA14BbjZ3Td2/w/MbJaZtZpZ6/r162P/EBKXpkAoh6Nx9KEKo250jqVFSKLv6czu/g72ts9EYDfwPqAZ+EczO2yPHd3vdPcWd28ZOXJkQEjSL2Yq3ZTBHRXpAxVKN0lHIUUhib4NOKTT/VHAq73tE5VpDgA2Ap8EHnP3Xe7+BvA7oKW/QUv/uEo3ZSksDp50FNXBzFS6SZGQRP80MMbMms1sIHABMK/bPvOAGdHt6cCT7u4UyjWnWcEQ4CTgxcqELuUzcvoUxueodBOoUKPXOZYWfSb6qOZ+OfA4sAJ4wN2Xmdn1ZjY12u1uYLiZrQK+DBSHYM4BmoClFH5hfN/dl1T4Z5C4NE1xWbTwSDhNgZAu9SE7uft8YH63bdd0ur2dwlDK7s/b0tN2SZoSfTk01004dcami66MzSJdMFUWd42jD1WYAiHpKKRIiT6jlOjj0xELp87YdFGizyC3OnUpliGv2SuDmVaYShUl+iwytPBIOTR7ZTB1xqaLEn0maZBgOdQZG84wDa9MESX6LFJnbFncNStjqJypTyNNlOizSFMglEULj4QzM426SREl+gwyjJzpUxhXYSnBpKOoDroyNl2U6DPIozVjPa8O2TgKaUuZPoQ6Y9NFiT6LomapWlzxFC6YSjqK6mC6MjZVlOizqNiiV6KPRaWbcDm16FNFiT7D8vndSYdQVbTwSDjDtJRgiijRZ5FKN2VRiz6cavTpokSfRaXSjTpj4ygsPKJMH6JQo5e0UKLPpKhFr1E3seTdNegmkIZXposSfRapM7Y8yvPBVLpJFyX6DLJSjV4t+jgKc90o1YfIqXSTKkr0WRQlq7xKN7FoHH04MzTqJkWU6DNJo27KkVfpJlihRp90FFKkRJ9FqtGXxXGVbgJp1E26KNFnkWnUTTk06CZcoTNWqT4tlOgzKUpX6oyNxbWUYDCVbtJFiT6LotKNOmPjU54PkzNNgZAmSvQZpOGV5dHCI+E0jj5dlOizSJ2xZdFcN+E0TXG6KNFnkVr0ZdHsleEMtJRgiijRZ1KUrPRJjMUdcvrEBDFDq4OniE7bLCqWbvRJjKXwe1Et+hA5lW5SRYk+g6w0BYIWHonHVaMPVJgCIekopEiJPou08EhZdMFUOMN0fqWIEn0GWVS60fi3eLTwSDgzlejTJCjRm9kkM3vJzFaZ2eweHm8wsx9Hjy8ys9GdHhtnZgvNbJmZvWBmjZULX8rhmr2yLHlX6SaUmal0kyJ9JnozqwPmAJOBscAnzGxst90uAd5y9yOAW4Abo+fWAz8ALnX3Y4FTgV0Vi17KVJwCQZ/EOFS6CWeg8ytFQlr0E4FV7r7a3XcC9wPTuu0zDbgnuv0QcLoVevzOBJa4+/MA7r7B3dUDmDDTBVNlcdfslaFyKt2kSkiiPxhY2+l+W7Stx33cvR3YDAwHjgTczB43s2fN7J97+g/MbJaZtZpZ6/r16+P+DBJXafZK/c6No7DCVNJRVAfTXDepEpLoezq1u7+Dve1TD3wY+L/Rvx8zs9P32NH9TndvcfeWkSNHBoQk/VGa60ZtrlgKpRtl+hCavTJd6gP2aQMO6XR/FPBqL/u0RXX5A4CN0fb/cvc3AcxsPjAB+HU/45b+KJZu1KKPxWu4M3bXrl20tbWxffv2irzezGMHsDtfz4oVKyryetKhsbGRUaNGMWDAgODnhCT6p4ExZtYMrAMuAD7ZbZ95wAxgITAdeNLd3cweB/7ZzAYDO4FTKHTWSpJK4+gTjqPKOLXbGdvW1sZ+++3H6NGjK9IPsebNrezcnefI9+5XgeikyN3ZsGEDbW1tNDc3Bz+vz9JNVHO/HHgcWAE84O7LzOx6M5sa7XY3MNzMVgFfBmZHz30L+HcKvywWA8+6+y9i/FyyD3SMo9fwyjgKc93UZqrfvn07w4cPr1hnc61+80mamTF8+PDY37xCWvS4+3xgfrdt13S6vR04r5fn/oDCEEtJC81eWZZan49eI4qqQznvk66MzSQNryyH5jSLSadXaijRZ5AVyw9q0cejUTexKM+nhxJ9BnXMXqmPYhyOU6Ml+tS49dZbOeaYYxg2bBg33HBDWa+xadMmbr/99gpH1rNTTz2V1tbWirzWiy++yPjx4zn++ON55plnKvozKNFnkKPO2HJoKcFw5R6m22+/nfnz5/PWW28xe/Ye02oFeTcTfSX953/+J9OmTeO5555j+PDhFf0ZgjpjpbaYpikuS6EztvYz/XU/W8byV//ar9fY0Z4nn3cGDawDYOz79udf/v7YvT7n0ksvZfXq1UydOpWLL76Yl19+me985ztcdNFF7L///rS2tvKXv/yFb37zm0yfPh2Am266iQceeIAdO3bwsY99jOuuu47Zs2fz8ssvM378eM444wymTJnCzTffzM9//nMALr/8clpaWrjooosYPXo0M2bM4Gc/+xm7du3iwQcf5Oijj2br1q18/vOf54UXXqC9vZ1rr72WadOmsW3bNmbOnMny5cs55phj2LZtW68/z+7du7nkkktobW3FzLj44ov50pe+xOLFi7n00kt55513OPzww5k7dy4LFy7kW9/6FnV1dTz11FO8973v7fIz3HTTTf16P5ToM6hjrhu16OPQFAjxxG1G3HHHHTz22GP85je/KSXlotdee43//u//5sUXX2Tq1KlMnz6dJ554gpUrV/LHP/4Rd2fq1Kk89dRT3HDDDSxdupTFixcDsGDBgr3+vyNGjODZZ5/l9ttv5+abb+Z73/se//qv/8ppp53G3Llz2bRpExMnTuSjH/0o3/3udxk8eDBLlixhyZIlTJgwodfXXbx4MevWrWPp0qVA4ZsGwIUXXshtt93GKaecwjXXXMN1113Ht771LS699FKampq48sorWbNmTZefob+U6LOomK3Uoo8lK7NX9tXyDvHKxnd4Z2c7R//N/hWICM455xxyuRxjx47l9ddfB+CJJ57giSee4Pjjjwdgy5YtrFy5kkMPPTTWa//DP/wDACeccAI/+clPSq89b948br75ZqBwncErr7zCU089xRe+8AUAxo0bx7hx43p93cMOO4zVq1fz+c9/nilTpnDmmWeyefNmNm3axCmnnALAjBkzOO+8HkemV5QSfQYVR92odBOueKw01jxMpY9SQ0ND6XbxvXB3rrrqKj7zmc902XfNmjVd7tfX13dZe6H7xUbF166rq6O9vb302g8//DBHHXXUHrGEngPDhg3j+eef5/HHH2fOnDk88MAD3HJLMhMDqDM2kzTXTVzF34nK8zHs43bEWWedxdy5c9myZQsA69at44033mC//fbj7bffLu33/ve/n+XLl7Njxw42b97Mr3/d91RbZ511Frfddlvpl8pzzz0HwEc+8hHuu+8+AJYuXcqSJUt6fY0333yTfD7Pueeey9e+9jWeffZZDjjgAIYNG8Zvf/tbAO69995S676z7j9Df6lFn0GlGr1GOgcrHqksdMZWizPPPJMVK1bwt3/7twA0NTXxgx/8gMMPP5yTTz6ZD3zgA0yePJmbbrqJ888/n3HjxjFmzJhSqWdvrr76ar74xS8ybtw43J3Ro0fz85//nMsuu4yZM2cybtw4xo8fz8SJE3t9jXXr1jFz5szSt4lvfOMbANxzzz2lztjDDjuM73//+3s8d/jw4Xv8DP1hafv63tLS4pUalyo9W7LgYcYtuJgXz36IoyeekXQ4VaF9d54jvvooXz7jSL5w+pikw6m4FStWcMwxx1Ts9dZufIctO9o55qDK1Oilq57eLzN7xt1betpfpZsMMnXGxlY8UrpgKowOU7qodJNFVhjbrNJNuLw6Y+PJ2GH60Ic+xI4dO7psu/feeznuuOMSiqgrJfoMKjXo8xpHH0pffuLL0iFbtGhR0iHslUo3WaQrY8umBn0MOr1SQ4k+gywq3aiZGq54qHLK9EF0lNJFiT6LSi16jaMPVarRJxyHSDmU6DNIo27iK42jV6YPZMSp3ZQ74+TZZ59dmkOmv5qamiryOmmkRJ9BlitOapZwIFWkNAWC2vRhLF6JvrdEv3v33r91zp8/n6FDh8YMLns06iaDSslKpZtgatHHE/cwdZ5aeMCAATQ1NXHQQQexePFili9fzjnnnMPatWvZvn07V1xxBbNmzQJg9OjRtLa2smXLFiZPnsyHP/xhfv/733PwwQfzyCOPMGjQIF5++WU+97nPsX79egYPHsxdd93F0UcfzZ/+9Cc++clP0t7ezqRJkyp/EFJEiT6LNOomtuKMzpkYR//obPjLC/16ieHtuzkg7zAwSjF/cxxM7n3FqM5TCy9YsIApU6awdOlSmpubAZg7dy4HHngg27Zt48QTT+Tcc89l+PDhXV5j5cqV/OhHP+Kuu+7i/PPP5+GHH+ZTn/oUs2bN4o477mDMmDEsWrSIz372szz55JNcccUVXHbZZVx44YXMmTOnXz9v2inRZ1GuuMKUEn2o4sVlGUjzqTBx4sRSkofCEoM//elPAVi7di0rV67cI9E3Nzczfvx4oDDl8Jo1a9iyZQu///3vu0wFXLyw6Xe/+x0PP/wwAJ/+9Kf5yle+sk9/piQp0WdQsXSjhUfCZWr2yr20vENt2LSNjVt38oGDDyjr+UOGDCndXrBgAb/61a9YuHAhgwcP5tRTT91jqmHoOpVxXV0d27ZtI5/PM3To0F4X8MjENzTUGZtJlovG0aNEH6pjrptsJIb+inuU9jYt7+bNmxk2bBiDBw/mxRdf5A9/+EPw6+6///40Nzfz4IMPAoVy5fPPPw/AySefzP333w9Qmnq4VinRZ1Bpzdi8SjehOhYeSTiQahHzOHWelvef/umfujw2adIk2tvbGTduHFdffTUnnXRSrNe+7777uPvuu/ngBz/IscceyyOPPALAt7/9bebMmcOJJ57I5s2b4wVcZVS6ySKNo4+t+DtReX7f+eEPf9jj9oaGBh599NEeHyuuJjVixIjS2qwAV155Zel2c3Mzjz322B7PbW5uZuHChaX7s2fPLifsqqAWfQZpcfD4SjN9qkkfTM2I9FCizyDT8Mr4SnPdJBuGSDmU6LMoatGbOmODZWEpwUr+4jdQk34fKed9UqLPoFxOLfq48jXeGdvY2MiGDRsqeE7U6IFKmLuzYcMGGhsbYz1PnbFZVKzRa9RNMK/xzthRo0bR1tbG+vXrK/J6f922i7e3t1P/9qCKvJ50aGxsZNSoUbGeE5TozWwS8G2gDvieu9/Q7fEG4D+AE4ANwMfdfU2nxw8FlgPXuvvNsSKUiuu4SESlm1C1PtfNgAEDulyJ2l///sRL3PrkWtbcMKVirynl67N0Y4VVKuYAk4GxwCfMbGy33S4B3nL3I4BbgBu7PX4L0PP4KHnXFUfdaHhlONeasbGowz9dQmr0E4FV7r7a3XcC9wPTuu0zDbgnuv0QcLpF77SZnQOsBpZVJmTpNw2vjK3WSzeVpks10iUk0R8MrO10vy3a1uM+7t4ObAaGm9kQ4CvAdXv7D8xslpm1mllrpWqE0rtSo1SfwmAdc90o1YcozaeUcBxSEJLoezqzu79/ve1zHXCLu2/Z23/g7ne6e4u7t4wcOTIgJOmP0lw3SvTBNHtlPLlSi17nWBqEdMa2AYd0uj8KeLWXfdrMrB44ANgIfAiYbmbfBIYCeTPb7u7f6XfkUjbNXhlfaXFwDUgOUvzio4Fd6RCS6J8GxphZM7AOuAD4ZLd95gEzgIXAdOBJL/wq/9/FHczsWmCLknzy1KKPL6+lBGMpdcaqeJMKfSZ6d283s8uBxykMr5zr7svM7Hqg1d3nAXcD95rZKgot+Qv2ZdDSPx0jItSiD1XrwysrTZ2x6RI0jt7d5wPzu227ptPt7cB53Z/Xbf9ry4hP9oWO3thEw6gmSljxdJQHEw5EAE2BkEm5qNBs+hTGUDhWWngkTKkzVo2JVFCizyKVbmIrzUevPB9EnbHpokSfQaYCamwdF0wp04foKN3oHEsDJfoMykWjbvQhDFcaR688H6TUlkg2DIko0WdQqVWq0k2w0jh6JfogHesSJxyIAEr02ZTTqJu48qVvP8r0ITrOMJ1jaaBEn0GavTI+V2dsLDl1A6WKEn0GdXTG6nt1XMrzYYrnWF6ZPhWU6DPIStMU60MYqqNGr1QfQp2x6aJEn0G50sxc+hiG0qibeDqm2Ug4EAGU6LOpVKNX6SaULpiKp2PJA2X6NFCizyDLqTM2LtfslbHkSrNXShoo0WeQOmPjKyUs5fkgHVMgKNWngRJ9Bplmr4xNnbHxaLXKdFGiz6CcSjexdZRuJIRKN+miRJ9BGl4ZnxYeialYutH0lamgRJ9BqtHHp9kr49FRShcl+gwyjaOPrfjtR5OahclpHH2qKNFnkGkcfWx5zWkWi0bdpIsSfQZp4ZH4SlfGKtMH0RQI6aJEn0HFhUckBl0ZG0tH6UapPg2U6DNInbHxFdOVxtHHo0E36aBEn0Eq3cRXrDUrz4fRRXnpokSfQRp1E1/H8EoJoYVH0kWJPqPybirdxKALpuIpdlqrdJMOSvQZlcfU3IqhNAWCMn2QUote3xpTQYk+oxy16ONQ6Sae0jh6nWKpoESfUd7pb+lbxwpTSvVhipOa6RxLAyX6jHJyyvMxqEUfjzpj00WJPqMcMN+ddBhVQ/PRx6M1Y9NFiT6jXG3TWDTqJp6OUfTK9GkQlOjNbJKZvWRmq8xsdg+PN5jZj6PHF5nZ6Gj7GWb2jJm9EP17WmXDl3KpMzYeTc4Vj9a2SZc+E72Z1QFzgMnAWOATZja2226XAG+5+xHALcCN0fY3gb939+OAGcC9lQpc+qfQotenMJRrrptYOsbR6xxLg5AW/URglbuvdvedwP3AtG77TAPuiW4/BJxuZubuz7n7q9H2ZUCjmTVUInDpH9c4+piK89Er0wfR7JWpEpLoDwbWdrrfFm3rcR93bwc2A8O77XMu8Jy77+j+H5jZLDNrNbPW9evXh8Yu/ZA3tejjUIs+Hi08ki4hib6nU7v727fXfczsWArlnM/09B+4+53u3uLuLSNHjgwISfrPMNXog+VLwyuV6UOUOmOV6VMhJNG3AYd0uj8KeLW3fcysHjgA2BjdHwX8FLjQ3V/ub8BSGQ5qbsXQccFUwoFUCS08ki4hif5pYIyZNZvZQOACYF63feZR6GwFmA486e5uZkOBXwBXufvvKhW09F+eHPoYhtMFU/EUSzd5zWqWCn0m+qjmfjnwOLACeMDdl5nZ9WY2NdrtbmC4ma0CvgwUh2BeDhwBXG1mi6M/76n4TyFl0PDKODrG0SvVh9Bs9OlSH7KTu88H5nfbdk2n29uB83p43teBr/czRtkH8mqbxuJaeCQWXRmbLroyNqN0wVQ8Kt3E07GImTJ9GijRZ5jpi3UwzV4Zj0o36aJEn1F5cvpeHUPHpGbJxlEtcjmVbtJEiT7LVLoJpnH08RSPkqZASAcl+ozS7JXxqDM2Ho2jTxcl+oxycqrRx6AjFU/HqBsduTRQos+owpWxKt0EK9boVaQP0jEFQqJhSESJPqNck5rFUqw1K82HKbXodY6lghJ9RjmGqbkVTCtMxaM1Y9NFiT6jtPBIPK5RN7F0LDyScCACKNFnmBYeicNLC48kHEiV0JWx6aJEn1GOYagzNlSpZapEH6SY6NWiTwcl+oxyU4s+llJnrDJ9CNMkCKkSNHul1B4nx5gtf2TV1yYkHUoXb46YyEmX3ZF0GHtQZ2w8uagJ+fVfrOC2J1clG0wnOTP+8cwjOfWobM2WrkSfUX856lO89ecFSYfRxXu2vcyY1+f3vWMCOua6UaYPMXr4EM6dMIrN23YmHUoX//U/61nw0nolesmGD318Nh3rw6TDH+74LB987cGkw+iRaxx9LI0D6vi38z+YdBh7+F/f+DVbd7QnHca7TjV6SQ1v2I9BtpP2XelqBUKnSc2U6ataU2M9W5ToRZJjDU0AbH17c8KR7Klj0I0yfTUb0qBEL5Kousb9AXjn7Y0JR7In1xJTNaFJiV4kWXWDCol++5ZNCUfSO10wVd2aGurZsl2JXiQx9VGi37H1rwlHsqfSpGYq0le1poZ6dcaKJGngkAMA2Lk1hTV6VW5qQlNjPW8r0Yskp7FpKAC7tr2dcCR70gVTtaHYos/aHDxK9JIajVGLfve29LbodcFUdWtqqCfvsG3X7qRDeVcp0UtqDNpvGAD5Helr0WuR69owpKFwjWjWOmSV6CU1hjQVOmN9e/oSfZEa9NVtv8ZCos9anV6JXlKjfsBA3vEGbOeWpEPZg2v2ypowZGAh0Wdt5I0SvaTKOzYI25m+Fn1HjT7ZOKR/mhpVuhFJ3DYbTP2u9LXoO+a6UaavZk3FGr1a9CLJ2Z4bTH371qTD2ENxKUGl+eqmRC+SAjvrBjMgjYles1fWhFLpJmOJPmg+ejObBHwbqAO+5+43dHu8AfgP4ARgA/Bxd18TPXYVcAmwG/iCuz9eseil5uysb+KYrU+z5voPJB1KF1PzzpSBjtmUpEORfii26G/99UruXfjnhKPZ06lHjeSrU8ZW/HX7TPRmVgfMAc4A2oCnzWyeuy/vtNslwFvufoSZXQDcCHzczMYCFwDHAu8DfmVmR/ZqUgUAAAWPSURBVLp7tq5WkGC5lpksfzad6+E0NQxIOgTpp8YBdVz+d0ew+s309QMBvHf/xn3yuiGfqInAKndfDWBm9wPTgM6JfhpwbXT7IeA7Vui1mgbc7+47gD+Z2aro9RZWJnypNR887Xw47fykw5AaduVZRyUdwrsupEZ/MLC20/22aFuP+7h7O7AZGB74XMxslpm1mlnr+vXrw6MXEZE+hST6nrqful8P3ts+Ic/F3e909xZ3bxk5cmRASCIiEiok0bcBh3S6Pwp4tbd9zKweOADYGPhcERHZh0IS/dPAGDNrNrOBFDpX53XbZx4wI7o9HXjSC9eMzwMuMLMGM2sGxgB/rEzoIiISos/OWHdvN7PLgccpDK+c6+7LzOx6oNXd5wF3A/dGna0bKfwyINrvAQodt+3A5zTiRkTk3WVpm4C/paXFW1tbkw5DRKSqmNkz7t7S02O6MlZEpMYp0YuI1LjUlW7MbD3Qn2uTRwBvViicd1s1xw6KP0nVHDso/kp4v7v3OD49dYm+v8ystbc6VdpVc+yg+JNUzbGD4t/XVLoREalxSvQiIjWuFhP9nUkH0A/VHDso/iRVc+yg+PepmqvRi4hIV7XYohcRkU6U6EVEalzNJHozm2RmL5nZKjObnXQ8IcxsjZm9YGaLzaw12nagmf3SzFZG/w5LOs4iM5trZm+Y2dJO23qM1wpujd6PJWY2IbnIe439WjNbFx3/xWZ2dqfHropif8nMzkom6g5mdoiZ/cbMVpjZMjO7Itqe+uO/l9ir4vibWaOZ/dHMno/ivy7a3mxmi6Jj/+No0keiSRx/HMW/yMxGJxk/AO5e9X8oTLb2MnAYMBB4HhibdFwBca8BRnTb9k1gdnR7NnBj0nF2iu0jwARgaV/xAmcDj1JYk+AkYFEKY78WuLKHfcdG51AD0BydW3UJx38QMCG6vR/wP1GcqT/+e4m9Ko5/dAybotsDgEXRMX0AuCDafgdwWXT7s8Ad0e0LgB8nee64e8206EvLHbr7TqC43GE1mgbcE92+BzgnwVi6cPenKMxO2llv8U4D/sML/gAMNbOD3p1I99RL7L0pLYHp7n8CiktgJsbdX3P3Z6PbbwMrKKzWlvrjv5fYe5Oq4x8dw+IiswOiPw6cRmHpVNjz2Bffk4eA06OlVRNTK4k+aMnCFHLgCTN7xsxmRdve6+6vQeEDArwnsejC9BZvtbwnl0eljbmdymSpjj0qBRxPoWVZVce/W+xQJcffzOrMbDHwBvBLCt8yNnlh6VToGmNvS6smplYSfdCShSl0srtPACYDnzOzjyQdUAVVw3vy/4DDgfHAa8C/RdtTG7uZNQEPA19097/ubdcetiX6M/QQe9Ucf3ff7e7jKaySNxE4pqfdon9TF3+tJPqqXLLQ3V+N/n0D+CmFE+j14lfs6N83koswSG/xpv49cffXow9wHriLjvJAKmM3swEUEuV97v6TaHNVHP+eYq+24w/g7puABRRq9EOtsHQqdI2xt6VVE1MriT5kucNUMbMhZrZf8TZwJrCUrssyzgAeSSbCYL3FOw+4MBr9cRKwuVhiSItuNeuPUTj+kMIlMKMa793ACnf/904Ppf749xZ7tRx/MxtpZkOj24OAj1LoZ/gNhaVTYc9j39PSqslJuje4Un8ojDL4Hwq1s68mHU9AvIdRGFnwPLCsGDOFWt6vgZXRvwcmHWunmH9E4Sv2Lgqtlkt6i5fC19c50fvxAtCSwtjvjWJbQuHDeVCn/b8axf4SMDkFx/7DFL7+LwEWR3/Orobjv5fYq+L4A+OA56I4lwLXRNsPo/ALaBXwINAQbW+M7q+KHj8s6fNHUyCIiNS4WindiIhIL5ToRURqnBK9iEiNU6IXEalxSvQiIjVOiV5EpMYp0YuI1Lj/D3byEOWwrtGcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title(\"LR\")\n",
    "plt.plot(trained_logs[\"lr\"] + finetuned_logs_soft[\"lr\"], label=\"finetuned_soft\")\n",
    "plt.plot(trained_logs[\"lr\"], label=\"trained\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hard Pruning\n",
    "- Actually remove filters and weights by creating a new, pruned model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prune_hard(model, percentage=0.5, min_filters_per_layer: int=1):\n",
    "    \"\"\"Create a new model with a pruned architecture\"\"\"\n",
    "    model = copy.deepcopy(model.cpu())\n",
    "    masks = get_prune_masks(model, percentage, min_filters_per_layer)\n",
    "    \n",
    "    mask_ix = 0\n",
    "    with torch.no_grad():\n",
    "        for layer_ix in range(len(model.features)):\n",
    "            layer = model.features[layer_ix]\n",
    "            if not isinstance(layer, (nn.Conv2d, nn.BatchNorm2d)):\n",
    "                continue\n",
    "            mask_as_indices = masks[mask_ix].nonzero().view(-1)\n",
    "            if isinstance(layer, nn.Conv2d):\n",
    "                previous_mask_as_indices = torch.tensor(range(layer.weight.shape[1])) if mask_ix == 0 else masks[mask_ix-1].nonzero().view(-1) \n",
    "                new_conv = nn.Conv2d(len(previous_mask_as_indices), len(mask_as_indices), kernel_size=layer.kernel_size, padding=layer.padding)\n",
    "                new_conv.weight = nn.Parameter(layer.weight[mask_as_indices, :, :, :][:, previous_mask_as_indices, :, :])\n",
    "                new_conv.bias = None if layer.bias is None else nn.Parameter(layer.bias[mask_as_indices])\n",
    "                model.features[layer_ix] = new_conv\n",
    "            if isinstance(layer, nn.BatchNorm2d):\n",
    "                new_bn = nn.BatchNorm2d(len(mask_as_indices))\n",
    "                new_bn.weight = nn.Parameter(layer.weight[mask_as_indices])\n",
    "                new_bn.bias = nn.Parameter(layer.bias[mask_as_indices])\n",
    "                new_bn.running_mean = layer.running_mean[mask_as_indices]\n",
    "                new_bn.running_var = layer.running_var[mask_as_indices]\n",
    "                model.features[layer_ix] = new_bn\n",
    "                mask_ix += 1\n",
    "        \n",
    "        lin_mask = masks[-1].bool()\n",
    "        old_lin = model.classifier[0]\n",
    "        new_lin = nn.Linear(lin_mask.sum(), old_lin.out_features)\n",
    "        new_lin.weight = nn.Parameter(old_lin.weight[:, lin_mask])\n",
    "        new_lin.bias = nn.Parameter(old_lin.bias)\n",
    "        model.classifier[0] = new_lin\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pruned Filter: 0.49644886363636365\n",
      "Architecture: [30, 64, 118, 128, 253, 255, 255, 417, 181, 5, 5, 5, 411]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABIQAAAEvCAYAAAA0MRq8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3df7DeVX0n8PenAYn8UCAEyxDYpBgBZQAxQpwwu1TWXUBHmGqsulpK2aZDqWu3tcr+aLfr2hnc2rrERV0sSHQUKlQXxmFtBWV2q4R6qamrAUsimNyBlUglK8W4Bc/+cb+Jl+SG+9zkub/yfb1m7jzf7/me53k+Dzncm/vOOedbrbUAAAAA0B8/M9sFAAAAADCzBEIAAAAAPSMQAgAAAOgZgRAAAABAzwiEAAAAAHpGIAQAAADQMwfNdgFJcswxx7SlS5fOdhkAAAAAB4z77rvv+621xRNdmxOB0NKlSzMyMjLbZQAAAAAcMKrqu3u7ZskYAAAAQM8IhAAAAAB6RiAEAAAA0DNzYg8hAAAAYH75h3/4h4yOjmbHjh2zXUrvLVy4MEuWLMnBBx888HMEQgAAAMCUjY6O5ogjjsjSpUtTVbNdTm+11vL4449ndHQ0y5YtG/h5ky4Zq6qTq2rDuK//W1W/WVVHV9UXq+rB7vGorn9V1dqq2lRV36iqs/bjcwEAAABz0I4dO7Jo0SJh0CyrqixatGjKM7UmDYRaa99urZ3ZWjszySuSPJXkc0muSnJXa215kru68yS5MMny7mtNko9MqSIAAABgXhAGzQ378ucw1U2lz0+yubX23SQXJ1nXta9Lckl3fHGST7Qx65McWVXHTbkyAAAAAKbFVPcQenOSm7rjF7XWHk2S1tqjVXVs1358kq3jnjPatT26P4UCAAAAc9en790y1Nd76zknDvX1psPdd9+dD3zgA/n85z8/4fXf//3fz+GHH553vetdM/J+UzHwDKGqel6S1ye5ZbKuE7S1CV5vTVWNVNXItm3bBi0DAAAAYGDPPPPMbJewz6az9qksGbswyV+31r7XnX9v51Kw7vGxrn00yQnjnrckySO7v1hr7brW2orW2orFixdPvXIAAACg1x5++OGccsopufTSS3P66afnjW98Y5566qksXbo0733ve3PuuefmlltuyXnnnZeRkZEkyfe///0sXbo0SXLjjTfmF37hF3LBBRdk+fLlefe7373rtf/iL/4ir3rVq3LWWWdl9erVefLJJ5MkX/jCF3LKKafk3HPPzWc/+9lJa9y4cWPOO++8/NzP/VzWrl27q/2SSy7JK17xirzsZS/Lddddt6v98MMPz+/93u/lnHPOyT333DPl9xvUVJaMvSU/XS6WJLcnuTTJ1d3jbePaf6Oqbk5yTpLtO5eW9cLIxwfrt+Ky6a0DAAAAeuDb3/52rr/++qxatSq/8iu/kg9/+MNJkoULF+Yv//IvkyQf/ehH9/r8DRs25Otf/3oOOeSQnHzyyXnHO96R5z//+Xnf+96XO++8M4cddlje//7354//+I/z7ne/O7/6q7+aL33pS3nxi1+cX/zFX5y0vgceeCBf/vKX88Mf/jAnn3xyrrjiihx88MG54YYbcvTRR+dHP/pRXvnKV+YNb3hDFi1alL//+7/Paaedlve+973ZsWNHli9fPqX3G9RAM4Sq6tAkr0kyPoq6OslrqurB7trVXfsdSb6TZFOSjyX59aFVCwAAADDOCSeckFWrViVJ3va2t+0KgQYNT84///y88IUvzMKFC/PSl7403/3ud7N+/fps3Lgxq1atyplnnpl169blu9/9bh544IEsW7Ysy5cvT1XlbW9726Sv/9rXvjaHHHJIjjnmmBx77LH53vfGFl6tXbs2Z5xxRlauXJmtW7fmwQcfTJIsWLAgb3jDG5Jkn95vUAPNEGqtPZVk0W5tj2fsrmO7921JrhxKdQAAAADPYfdbru88P+yww3a1HXTQQfnJT36SJNmxY8ez+h9yyCG7jhcsWJCnn346rbW85jWvyU033fSsvhs2bJjyLd4nev277747d955Z+65554ceuihOe+883bVtXDhwixYsGCvn29YpnrbeQAAAIA5Y8uWLbnnnnuSJDfddFPOPffcPfosXbo09913X5Lk1ltvnfQ1V65cma985SvZtGlTkuSpp57K3/7t3+aUU07JQw89lM2bN+96v32xffv2HHXUUTn00EPzwAMPZP369RP2G9b7TWSqt50HAAAA2MNs3Sb+1FNPzbp16/Jrv/ZrWb58ea644op86EMfelafd73rXXnTm96UT37yk3n1q1896WsuXrw4N954Y97ylrfkxz/+cZLkfe97X17ykpfkuuuuy2tf+9occ8wxOffcc/PNb35zyjVfcMEF+ehHP5rTTz89J598clauXDlhv4ULFw7l/SZSYyu8ZteKFSvazt2+5z2bSgMAANAD999/f0499dRZreHhhx/O6173uqGFJPPZRH8eVXVfa23FRP0tGQMAAADoGUvGAAAAgHlp6dKlc2J20Mc//vFcc801z2pbtWpVrr322lmqaHICIQAAAID9cNlll+Wyy+bX1jCWjAEAAAD0jEAIAAAAoGcEQgAAAAA9IxACAAAA5qUnnngiH/7wh6f8vIsuuihPPPHEUGo4/PDDh/I6M82m0gAAAMD+G/n4cF9vxeSbNO8MhH7913/9We3PPPNMFixYsNfn3XHHHftd3nwnEAIAAADmpauuuiqbN2/OmWeemYMPPjiHH354jjvuuGzYsCEbN27MJZdckq1bt2bHjh155zvfmTVr1iQZu139yMhInnzyyVx44YU599xz89WvfjXHH398brvttjz/+c/P5s2bc+WVV2bbtm059NBD87GPfSynnHJKHnroobz1rW/N008/nQsuuGCW/wvsO0vGAAAAgHnp6quvzkknnZQNGzbkD//wD/NXf/VX+YM/+INs3LgxSXLDDTfkvvvuy8jISNauXZvHH398j9d48MEHc+WVV+Zb3/pWjjzyyPzZn/1ZkmTNmjX50Ic+lPvuuy8f+MAHds1Ceuc735krrrgiX/va1/KzP/uzM/dhh8wMIQAAAOCAcPbZZ2fZsmW7zteuXZvPfe5zSZKtW7fmwQcfzKJFi571nGXLluXMM89MkrziFa/Iww8/nCeffDJf/epXs3r16l39fvzjHydJvvKVr+wKjd7+9rfnPe95z7R+pukiEAIAAAAOCIcddtiu47vvvjt33nln7rnnnhx66KE577zzsmPHjj2ec8ghh+w6XrBgQX70ox/lJz/5SY488shs2LBhwvepquEXP8MsGQMAAADmpSOOOCI//OEPJ7y2ffv2HHXUUTn00EPzwAMPZP369QO/7gte8IIsW7Yst9xyS5KktZa/+Zu/SZKsWrUqN998c5LkU5/61H5+gtkjEAIAAADmpUWLFmXVqlU57bTT8ju/8zvPunbBBRfk6aefzumnn57f/d3fzcqVK6f02p/61Kdy/fXX54wzzsjLXvay3HbbbUmSa665Jtdee21e+cpXZvv27UP7LDOtWmuzXUNWrFjRRkZGZruM4Rj0NnsD3D4PAAAA5qr7778/p5566myXQWeiP4+quq+1tmKi/mYIAQAAAPSMQAgAAACgZwRCAAAAAD0jEAIAAAD2yVzYl5h9+3MQCAEAAABTtnDhwjz++ONCoVnWWsvjjz+ehQsXTul5B01TPQAAAMABbMmSJRkdHc22bdtmu5TeW7hwYZYsWTKl5wiEAAAAgCk7+OCDs2zZstkug31kyRgAAABAzwiEAAAAAHpGIAQAAADQMwIhAAAAgJ4ZKBCqqiOr6taqeqCq7q+qV1XV0VX1xap6sHs8qutbVbW2qjZV1Teq6qzp/QgAAAAATMWgM4SuSfKF1topSc5Icn+Sq5Lc1VpbnuSu7jxJLkyyvPtak+QjQ60YAAAAgP0yaSBUVS9I8o+TXJ8krbX/11p7IsnFSdZ13dYluaQ7vjjJJ9qY9UmOrKrjhl45AAAAAPtkkBlCP5dkW5KPV9XXq+pPquqwJC9qrT2aJN3jsV3/45NsHff80a7tWapqTVWNVNXItm3b9utDAAAAADC4QQKhg5KcleQjrbWXJ/n7/HR52ERqgra2R0Nr17XWVrTWVixevHigYgEAAADYf4MEQqNJRltr93bnt2YsIPrezqVg3eNj4/qfMO75S5I8MpxyAQAAANhfkwZCrbX/k2RrVZ3cNZ2fZGOS25Nc2rVdmuS27vj2JL/U3W1sZZLtO5eWAQAAADD7Dhqw3zuSfKqqnpfkO0kuy1iY9JmqujzJliSru753JLkoyaYkT3V9AQAAAJgjBgqEWmsbkqyY4NL5E/RtSa7cz7oAAAAAmCaD7CEEAAAAwAFEIAQAAADQMwIhAAAAgJ4RCAEAAAD0jEAIAAAAoGcEQgAAAAA9IxACAAAA6BmBEAAAAEDPCIQAAAAAekYgBAAAANAzAiEAAACAnhEIAQAAAPSMQAgAAACgZwRCAAAAAD0jEAIAAADoGYEQAAAAQM8IhAAAAAB6RiAEAAAA0DMCIQAAAICeEQgBAAAA9IxACAAAAKBnBEIAAAAAPSMQAgAAAOgZgRAAAABAzwiEAAAAAHpGIAQAAADQMwIhAAAAgJ4RCAEAAAD0zECBUFU9XFX/u6o2VNVI13Z0VX2xqh7sHo/q2quq1lbVpqr6RlWdNZ0fAAAAAICpmcoMoZ9vrZ3ZWlvRnV+V5K7W2vIkd3XnSXJhkuXd15okHxlWsQAAAADsv/1ZMnZxknXd8bokl4xr/0Qbsz7JkVV13H68DwAAAABDNGgg1JL8RVXdV1VrurYXtdYeTZLu8diu/fgkW8c9d7RrAwAAAGAOOGjAfqtaa49U1bFJvlhVDzxH35qgre3RaSxYWpMkJ5544oBlAAAAALC/Bpoh1Fp7pHt8LMnnkpyd5Hs7l4J1j4913UeTnDDu6UuSPDLBa17XWlvRWluxePHiff8EAAAAAEzJpIFQVR1WVUfsPE7yz5J8M8ntSS7tul2a5Lbu+PYkv9TdbWxlku07l5YBAAAAMPsGWTL2oiSfq6qd/T/dWvtCVX0tyWeq6vIkW5Ks7vrfkeSiJJuSPJXksqFXDQAAAMA+mzQQaq19J8kZE7Q/nuT8CdpbkiuHUh0AAAAAQ7c/t50HAAAAYB4SCAEAAAD0jEAIAAAAoGcEQgAAAAA9IxACAAAA6BmBEAAAAEDPCIQAAAAAekYgBAAAANAzAiEAAACAnhEIAQAAAPSMQAgAAACgZwRCAAAAAD0jEAIAAADoGYEQAAAAQM8IhAAAAAB6RiAEAAAA0DMCIQAAAICeEQgBAAAA9IxACAAAAKBnBEIAAAAAPSMQAgAAAOgZgRAAAABAzwiEAAAAAHpGIAQAAADQMwIhAAAAgJ4RCAEAAAD0jEAIAAAAoGcEQgAAAAA9M3AgVFULqurrVfX57nxZVd1bVQ9W1Z9W1fO69kO6803d9aXTUzoAAAAA+2IqM4TemeT+cefvT/LB1tryJD9IcnnXfnmSH7TWXpzkg10/AAAAAOaIgQKhqlqS5LVJ/qQ7rySvTnJr12Vdkku644u783TXz+/6AwAAADAHDDpD6L8keXeSn3Tni5I80Vp7ujsfTXJ8d3x8kq1J0l3f3vUHAAAAYA6YNBCqqtcleay1dt/45gm6tgGujX/dNVU1UlUj27ZtG6hYAAAAAPbfIDOEViV5fVU9nOTmjC0V+y9Jjqyqg7o+S5I80h2PJjkhSbrrL0zyd7u/aGvtutbaitbaisWLF+/XhwAAAABgcJMGQq21f9NaW9JaW5rkzUm+1Fr7F0m+nOSNXbdLk9zWHd/enae7/qXW2h4zhAAAAACYHVO5y9ju3pPkt6pqU8b2CLq+a78+yaKu/beSXLV/JQIAAAAwTAdN3uWnWmt3J7m7O/5OkrMn6LMjyeoh1AYAAADANNifGUIAAAAAzEMCIQAAAICeEQgBAAAA9IxACAAAAKBnBEIAAAAAPSMQAgAAAOgZgRAAAABAzwiEAAAAAHpGIAQAAADQMwIhAAAAgJ4RCAEAAAD0jEAIAAAAoGcEQgAAAAA9IxACAAAA6BmBEAAAAEDPCIQAAAAAekYgBAAAANAzAiEAAACAnhEIAQAAAPSMQAgAAACgZwRCAAAAAD0jEAIAAADoGYEQAAAAQM8IhAAAAAB6RiAEAAAA0DMCIQAAAICeEQgBAAAA9IxACAAAAKBnJg2EqmphVf1VVf1NVX2rqv5j176squ6tqger6k+r6nld+yHd+abu+tLp/QgAAAAATMUgM4R+nOTVrbUzkpyZ5IKqWpnk/Uk+2FpbnuQHSS7v+l+e5AettRcn+WDXDwAAAIA5YtJAqI15sjs9uPtqSV6d5NaufV2SS7rji7vzdNfPr6oaWsUAAAAA7JeB9hCqqgVVtSHJY0m+mGRzkidaa093XUaTHN8dH59ka5J017cnWTTMogEAAADYdwMFQq21Z1prZyZZkuTsJKdO1K17nGg2UNu9oarWVNVIVY1s27Zt0HoBAAAA2E9TustYa+2JJHcnWZnkyKo6qLu0JMkj3fFokhOSpLv+wiR/N8FrXddaW9FaW7F48eJ9qx4AAACAKRvkLmOLq+rI7vj5Sf5pkvuTfDnJG7tulya5rTu+vTtPd/1LrbU9ZggBAAAAMDsOmrxLjkuyrqoWZCxA+kxr7fNVtTHJzVX1viRfT3J91//6JJ+sqk0Zmxn05mmoGwAAAIB9NGkg1Fr7RpKXT9D+nYztJ7R7+44kq4dSHQAAAABDN6U9hAAAAACY/wRCAAAAAD0jEAIAAADoGYEQAAAAQM8IhAAAAAB6RiAEAAAA0DMCIQAAAICeEQgBAAAA9IxACAAAAKBnBEIAAAAAPSMQAgAAAOgZgRAAAABAzwiEAAAAAHpGIAQAAADQMwIhAAAAgJ4RCAEAAAD0jEAIAAAAoGcEQgAAAAA9IxACAAAA6BmBEAAAAEDPCIQAAAAAekYgBAAAANAzAiEAAACAnhEIAQAAAPSMQAgAAACgZwRCAAAAAD0jEAIAAADoGYEQAAAAQM8IhAAAAAB6ZtJAqKpOqKovV9X9VfWtqnpn1350VX2xqh7sHo/q2quq1lbVpqr6RlWdNd0fAgAAAIDBDTJD6Okkv91aOzXJyiRXVtVLk1yV5K7W2vIkd3XnSXJhkuXd15okHxl61QAAAADss0kDodbao621v+6Of5jk/iTHJ7k4ybqu27okl3THFyf5RBuzPsmRVXXc0CsHAAAAYJ9MaQ+hqlqa5OVJ7k3yotbao8lYaJTk2K7b8Um2jnvaaNe2+2utqaqRqhrZtm3b1CsHAAAAYJ8MHAhV1eFJ/izJb7bW/u9zdZ2gre3R0Np1rbUVrbUVixcvHrQMAAAAAPbTQIFQVR2csTDoU621z3bN39u5FKx7fKxrH01ywrinL0nyyHDKBQAAAGB/DXKXsUpyfZL7W2t/PO7S7Uku7Y4vTXLbuPZf6u42tjLJ9p1LywAAAACYfQcN0GdVkrcn+d9VtaFr+7dJrk7ymaq6PMmWJKu7a3ckuSjJpiRPJblsqBUDAAAAsF8mDYRaa3+ZifcFSpLzJ+jfkly5n3UBAAAAME2mdJcxAAAAAOY/gRAAAABAzwiEAAAAAHpGIAQAAADQMwIhAAAAgJ4RCAEAAAD0jEAIAAAAoGcEQgAAAAA9IxACAAAA6BmBEAAAAEDPCIQAAAAAekYgBAAAANAzAiEAAACAnhEIAQAAAPSMQAgAAACgZwRCAAAAAD0jEAIAAADoGYEQAAAAQM8IhAAAAAB6RiAEAAAA0DMCIQAAAICeOWi2C2AOG/n4YP1WXDa9dQAAAABDJRACDgwCTAAAgIEJhNh/fhEHAACAecUeQgAAAAA9IxACAAAA6BmBEAAAAEDP2EOobwbd72c+vLc9iUjy6Xu3JElO2vJ3e+1zzrKjZ6ocAACAeWHSQKiqbkjyuiSPtdZO69qOTvKnSZYmeTjJm1prP6iqSnJNkouSPJXkl1trfz09pQPsg9ncBN0G7AAAwBwxyAyhG5P81ySfGNd2VZK7WmtXV9VV3fl7klyYZHn3dU6Sj3SPQF/MwdDj3of2PntovGfNJJqDn2Pa9fEzAwBAT00aCLXW/mdVLd2t+eIk53XH65LcnbFA6OIkn2ittSTrq+rIqjqutfbosAoGmFNmcxkmAADAPtrXPYRetDPkaa09WlXHdu3HJ9k6rt9o1yYQAvbNJIHLc+0dNB0GmW1kz6LOVMIys44AAGBGDfsuYzVBW5uwY9WaqhqpqpFt27YNuQwAAAAA9mZfZwh9b+dSsKo6LsljXftokhPG9VuS5JGJXqC1dl2S65JkxYoVE4ZGwAFsDi61GnSvoQOV2U8AANAf+xoI3Z7k0iRXd4+3jWv/jaq6OWObSW+3fxAcAOZgeDOXCVYAAIC5bpDbzt+UsQ2kj6mq0ST/IWNB0Geq6vIkW5Ks7rrfkbFbzm/K2G3nbQoxU/zCDgAAAAxokLuMvWUvl86foG9LcuX+FgUc+MyiGQK3iQcAAPbRvi4Zg9nnl+ED3oG8p89zfraH/ijJ/A7EPn3vlindAW7zM1smbH/rOScOqyQAAGAcgRDAHDVoIDafgyMAAGB2CIQADnTD3mNsgNebyuygsf63THxhwW5hlxl/AAAwFAIhgB4Y1vK7WV/GZ6koAAAMhUAIYJ6b9ZAGAACYd35mtgsAAAAAYGaZITTXDXvvDxjP+GKOG2T2k021AQBg6swQAgAAAOgZM4Q48NmEFgAAAJ5FIAQHollcCmaDYwAAgLlPIATAvDZRCLn5mS3POn/rOSfOVDkAADAvCIRgPrEJNAzkpC23PLthwV42nh50qailpwAAHGAEQrDTVMIWv/QBAAAwj7nLGAAAAEDPmCEEAJ1P37tlwvaTtvx0n6Jzlu1l+RkAAMwjAqHZYi8YgBmz17vfPfRHzzo9aX9eKz/dzNom1gAAzHWWjAEAAAD0jEAIAAAAoGcsGYN94RbUAAAAzGMCIQBgT4JvAIADmkAI5gKbjMMB4aQttyRJ9nKzsl02n7h6xjee3nkHtZ01TsQd1AAA+kMgBAAz7KQttyQLBgxfZnAGznPdQW0noREAwIFBIATTycwfYC8GCV+SJA/90aRdBglpTtoy4PsBANAL7jIGAAAA0DNmCAHAPDfwbKNpeK/Nz0y8YdIeeyRNMGNyv5eo2dB672wKDgBMQiAEAMxZzxUa7QyjJt2guwtHhrZH0jwNUSb6/LsHejO92TkAMHsEQgDAPtvrXcsG3TR7jhkoNFoxA4VM0ad3u7WdPaMAgMkIhACAoZuJZWw7w6h7J161Nm3uvWU4G30P+t9ovm4avntINd7OP7tJP9s8nY0FAPPBtARCVXVBkmuSLEjyJ621q6fjfQAA5qJhBmLTGa7tMcNrbzO7Bg1mxu1dNEhINdln2/zMFsvYAGCaDD0QqqoFSa5N8poko0m+VlW3t9Y2Dvu9AACYAYNuUg0AzBvTMUPo7CSbWmvfSZKqujnJxUkEQsxZw5y2DwDz1UzesW4QJ225ZahLAs9ZdvRQlqGNXw63t3209vg7g+VvAMwx0xEIHZ9k67jz0STnTMP7wIz/xXVYd6iZ6boFWQDQ/fx9aPI9oCZz0qDvNc7ud3TbaY8lcRPMxpqpvzdsPnF1ksHu3Desmqby96ad9e2N5YXz056b4s+DkHXA/wc2n7h6Rsflzv+We73hQ2fXf8s+BtUD/tmds/q3Z6CY2VetteG+YNXqJP+8tfYvu/O3Jzm7tfaO3fqtSbKmOz05ybeHWsjcdEyS7892ETAJ45T5wDhlPjBOmQ+MU+Y6Y5T5YC6P03/UWls80YXpmCE0muSEcedLkjyye6fW2nVJrpuG95+zqmqktTYHb1YLP2WcMh8Yp8wHxinzgXHKXGeMMh/M13H6M9Pwml9LsryqllXV85K8Ocnt0/A+AAAAAOyDoc8Qaq09XVW/keTPM3bb+Rtaa98a9vsAAAAAsG+mY8lYWmt3JLljOl57nuvVEjnmLeOU+cA4ZT4wTpkPjFPmOmOU+WBejtOhbyoNAAAAwNw2HXsIAQAAADCHCYSmQVVdUFXfrqpNVXXVBNcPqao/7a7fW1VLZ75K+m6AcfpbVbWxqr5RVXdV1T+ajTrpt8nG6bh+b6yqVlXz7u4OzG+DjNGqelP3/fRbVfXpma4RBviZf2JVfbmqvt793L9oNuqk36rqhqp6rKq+uZfrVVVru3H8jao6a6ZrpN8GGKP/ohub36iqr1bVGTNd41QJhIasqhYkuTbJhUlemuQtVfXS3bpdnuQHrbUXJ/lgkvfPbJX03YDj9OtJVrTWTk9ya5L/PLNV0ncDjtNU1RFJ/lWSe2e2QvpukDFaVcuT/Jskq1prL0vymzNeKL024PfSf5/kM621l2fsDsEfntkqIUlyY5ILnuP6hUmWd19rknxkBmqC8W7Mc4/Rh5L8k+73p/+UebCvkEBo+M5Osqm19p3W2v9LcnOSi3frc3GSdd3xrUnOr6qawRph0nHaWvtya+2p7nR9kiUzXCMM8v00GfuB+5+T7JjJ4iCDjdFfTXJta+0HSdJae2yGa4RBxmlL8oLu+IVJHpnB+iBJ0lr7n0n+7jm6XJzkE23M+iRHVtVxM1MdTD5GW2tf3fnzPvPk9yeB0PAdn2TruPPRrm3CPq21p5NsT7JoRqqDMYOM0/EuT/I/prUi2NOk47SqXp7khNba52eyMOgM8r30JUleUlVfqar1VfVc/7II02GQcfr7Sd5WVaMZu1PwO2amNJiSqf79FWbTvPj9aVpuO99zE8302f1WboP0gek08BisqrclWZHkn0xrRbCn5xynVfUzGVt2+8szVRDsZpDvpQdlbHnDeRn7l8L/VVWntdaemObaYKdBxulbktzYWvujqnpVkk924/Qn018eDMzvUMwLVfXzGQuEzp3tWiZjhtDwjSY5Ydz5kuw57XZXn6o6KGNTc59reiQM2yDjNFX1T5P8uySvb639eIZqg50mG6dHJDktyd1V9XCSlUlut7E0M2jQn/m3tdb+obX2UJJvZywggpkyyDi9PMlnknOU7nYAAAGpSURBVKS1dk+ShUmOmZHqYHAD/f0VZlNVnZ7kT5Jc3Fp7fLbrmYxAaPi+lmR5VS2rqudlbGO+23frc3uSS7vjNyb5UmtNus1MmnScdktx/lvGwiB7XjAbnnOctta2t9aOaa0tba0tzdha7de31kZmp1x6aJCf+f89yc8nSVUdk7ElZN+Z0Srpu0HG6ZYk5ydJVZ2asUBo24xWCZO7PckvdXcbW5lke2vt0dkuCnaqqhOTfDbJ21trfzvb9QzCkrEha609XVW/keTPkyxIckNr7VtV9d4kI62125Ncn7GpuJsyNjPozbNXMX004Dj9wySHJ7ml2/N8S2vt9bNWNL0z4DiFWTPgGP3zJP+sqjYmeSbJ78yHfzHkwDHgOP3tJB+rqn+dsSU4v+wfK5lpVXVTxpbXHtPtZ/UfkhycJK21j2Zsf6uLkmxK8lSSy2anUvpqgDH6exnbG/jD3e9PT7fW5vTM9fK9HgAAAKBfLBkDAAAA6BmBEAAAAEDPCIQAAAAAekYgBAAAANAzAiEAAACAnhEIAQAAAPSMQAgAAACgZwRCAAAAAD3z/wGq39xMk+ATiwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 1440x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pruned_model_hard = prune_hard(trained_model, percentage=0.5, min_filters_per_layer=5)\n",
    "print(\"Pruned Filter:\", 1 - count_filters(pruned_model_hard) / count_filters(trained_model))\n",
    "print(\"Architecture:\", count_filters(pruned_model_hard, cat=False))\n",
    "\n",
    "plt.figure(figsize=(20, 5))\n",
    "sns.distplot(get_gammas(pruned_model_hard, cat=True), bins=100, kde=False, label=\"pruned_hard\")\n",
    "sns.distplot(get_gammas(trained_model, cat=True), bins=100, kde=False, label=\"trained\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pruned Parameters: 0.7706084381185592\n"
     ]
    }
   ],
   "source": [
    "print(\"Pruned Parameters:\", 1 - count_parameters(pruned_model_hard) / count_parameters(trained_model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/160 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reducing LR at: [80, 120]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Test Acc: 71.7000%: 100%|██████████| 160/160 [33:09<00:00, 12.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final Acc: 0.7170000076293945\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "finetuned_model_hard, finetuned_logs_hard = train(pruned_model_hard, n_epochs=160, batch_size=64, bn_l1_weight=0)\n",
    "print(\"Final Acc:\", finetuned_logs_hard[\"test_acc\"][-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dd5xcdbn48c8zbftmN1tSNmU3nZDeaQqCtOsNyAUpNn6WXBEEr14EfoqgXq7+aCqIoiJXlBIVFAICCSC5SAlhA+m9bHrZbG/Tv78/zuzsbJ/d7GbmTJ7365VX5pQ58+zM2We/83y/53vEGINSSin7cyQ6AKWUUgNDE7pSSqUITehKKZUiNKErpVSK0ISulFIpQhO6UkqlCE3oSimVIjShK9sRkZUiUiMiaYmORalkogld2YqIlALnAAZYfBJf13WyXkup/tKEruzmC8Aq4PfAF1tXikiGiDwgIntFpE5E3haRjMi2s0XkXRGpFZH9InJ9ZP1KEflKzDGuF5G3Y5aNiNwoIjuAHZF1P48co15E1ojIOTH7O0Xk/4rILhFpiGwfLSKPiMgDsT+EiLwoIt8cjDdInbo0oSu7+QLwVOTfRSIyLLL+fmAucCYwFPgOEBaRMcArwMNAETALWNuH17scWAhMjSx/EDnGUOBp4C8ikh7Z9i3gWuBSIBf4EtAMPAFcKyIOABEpBM4HnunLD65UbzShK9sQkbOBscCfjTFrgF3AdZFE+SXgFmPMQWNMyBjzrjHGB3wWeN0Y84wxJmCMqTLG9CWh/9gYU22MaQEwxjwZOUbQGPMAkAZMjuz7FeB7xphtxrIusu9qoA4riQNcA6w0xhw9wbdEqXY0oSs7+SKwwhhzPLL8dGRdIZCOleA7Gt3N+njtj10QkW+LyJZIWacWGBJ5/d5e6wngc5HHnwP+eAIxKdUl7ehRthCph38GcIrIkcjqNCAPGAF4gfHAug5P3Q8s6OawTUBmzPLwLvaJTkcaqZffhtXS3mSMCYtIDSAxrzUe2NjFcZ4ENorITOA04PluYlKq37SFruziciCEVcueFfl3GvBPrLr648CDIjIy0jl5RmRY41PABSLyGRFxiUiBiMyKHHMtcIWIZIrIBODLvcSQAwSBSsAlIt/HqpW3egz4kYhMFMsMESkAMMYcwKq//xF4rrWEo9RA0oSu7OKLwP8YY/YZY460/gN+gVUnvx3YgJU0q4H/BziMMfuwOim/HVm/FpgZOeZPAT9wFKsk8lQvMSzH6mDdDuzF+lYQW5J5EPgzsAKoB34HZMRsfwKYjpZb1CARvcGFUieHiHwMq/RSaowJJzoelXq0ha7USSAibuAW4DFN5mqwaEJXapCJyGlALVbn7c8SHI5KYVpyUUqpFKEtdKWUShEJG4deWFhoSktLE/XySillS2vWrDlujCnqalvCEnppaSnl5eWJenmllLIlEdnb3TYtuSilVIrQhK6UUilCE7pSSqUITehKKZUiNKErpVSK0ISulFIpQhO6UkqlCE3oypaCAT/1tVVsXvUqe7esIRQM8sHzv2DPpvd7fJ7f56ViSzne5kaOH9nHsYN72LX+XUxY58tS9qd3LFJJIxQMsv6Np/Ad24Xz+BbCI+fiyinCmDBse5WRDetxGEO2aSQsDnJpYioQNkKV5DGfGurXZrL7uWL8jgwaskaT1XKYgDODoCuL3Ob95IRqKeU4YSMUijWPUTFQ89ccjjuHUTX+ckrGTGD02dcm9L1Qqj80oauEMeEw5ct+RbBmH4SD5BwrZ7bPun9zLdnk1a2I7ttiPGzNXoARB4G0fCQcIlwwAVdeCYHDm3E37Gf36DMZuvVp/M5MBMPYunKqXcPIClWR6a2gylNCszufirFfxjQcQbIKMP5mJD0Xx7FNjKr9gInb72fr3umgCV3ZkCb0U1g4FMLhdCbs9cuX/ZL5a78bXa4ji1WTb2PS+V8kv3AEh/fvwNfcAMCwMZOZnZUTx1G/3W6pOObxyF6eGQz4ufLnr5KRlaO3FFK2pAn9FHKoYht7//EYZfv+SsXwi5h65G9snXYrjn3vMPvmP+FwOFj9l3sxviamXHoj2974PVklU5l29uJOx2qsryErewji6F83THNjHZPX3sNm9zTG/cdyPGkZDHE6WRSzz4ixk/v5k/aPy+3BlVuET8vpyqY0oZ8iypf9ijlr7mCkGALGyfAj1u0zF2z8gbX9oasZ3rCRheYIAE0P/4aF4iWw2clHTT9lytlXcGD7R1S9vxTjzmTevsfZ4xxN4NKfEvS1cNqii9u19hvra6jcv52y0xd2Gc/G1/7AAprh/DtJz8we5J8+fg4RQmhGV/akCf0UsPb1Z5i+5k62pE3Dc+mPqd39IVPX/Td7PROYGthIHVnMq3+d7a5JvD/xS3j2/i+Z/uOEP/kj5LW7mP7uN/G/eysTxcfEyDF3ucrICdVR/NKnAVi14bN4ys6goHQ6x7a8g2v7S8xoeo8Pz3yYORd9vl083uZG8jc9wX4ZyWkLLjzJ70bPRCCs93xRNqUJPcXVHj/C1H/eyF73OEqWPEte4XCYdQ7eS75CSUsz69a+CYTxr/sr05b8jklZOcBt0ec3nraA9b/+AiF3FjL+PMbOvYj961cyfsG/0FRXxQfPf590byWLjjwFR56C92Bs5LkNZDDl3W+zc/g4av7xc8KuDIwrgwlHX2EitXww678Z3c+SzWARBL2L1+DaeayB4UMyyE47uelnb1UTyzcd4RNThuFxOjja4KW2OUBJXgajh2YgIrT4QxTlpHV7DGMMmw7VMzTLQ8XxJkbkZeB2CsGQobQwi3DYYACHwPt7qinM9jC+KJsj9V6cDmFvVTPDc9MZlW+93kBL2C3o5s2bZ3Q+9MG3d9taxj7zccrn3ce8Ty0ZlNdoaqhl04rfI04Xjl1vEEzLY1h1OfKZJxj6zKXkSEu7/ddlLIQFX2XmeVcNSjwn4vO/e59GX5C/ff2sRIdie1WNPpr9IQqyPfz89R3MHpPHqPxMFv/ibQqz0zh7QiE3nz+R0sKsQY/FGMPVv17F6orqLrcXZHnwB8M0+IJcOXcU37xgIt945iPOHF/ArNH53PXCRhaNL2DM0Ex+9vqOTs/3uBw8dM1sHv3fXeyvbmZolocdxxpJczkYmuXhcJ233f7f/9RUvnR2Wb9+FhFZY4yZ19U2baGnuKDfOpGc7u5bHScqKyePBf/2zcjSTYA1JFEcDtaf+wuat/2DgoXXUP3O7wm7M1j01Yf73Zk62ERESy795A2EqKhqorrJzz+2HOOp9/fREgjhdAihsMHjcjAqP4P8TA9TRuSyfNMRNh2q5/kbzyIQDrP1cAPzS/MHrOUaDhueen8vH+6rxRsIsbqimpvOm8DYgkwAMjxOhuWmc7jOy2/f2k12mosJxdn8cdVeXlp/iLCBj/bVAlCSl8FL6w/jD4aZOzafS6YNZ9KwHI41+Gj2B/nVyl187ck1ZHmcLCgbij8U5gtnlrJqdxX+YJivfXw8YWMYmZfB0XovZ44vGJCfsSNN6CkuFLASurgGL6F3pTVhzzjvSjjvSmvl7I+d1Bj6QwC05NIvtz+3nufXHgLA7RQ+OXUY88YO5Xijj5mj8/jhi5tp8Ye476oZfGLKMP53eyXX/89qFv/ibfZWN+MPhjlnYiGhsOGR6+aQn+Wh2R+kyddzGaQ7f/voIHe+sIlhuWk4RLh63mhuuWAibmfnxsTimdagVmOs0smuykaunT8GgM2H6/jk1OGsP1DLfcu38eMrpjNpWPshtGeOL6C8ooZzJxczfEh6dP3nF43lZIoroYvIxcDPASfwmDHmJx22/xQ4L7KYCRQbY/IGMlDVP6GADxjcFnoqcQhoOu+7jQfreH7tIa6YU8Kl00awaHxBpxr5BacNwyFEW+Afn1TEz66exf/96wb+dcZIPC4Hz6zehwjcv2IbC8qGcufzG2n0BZlQnE1epoebzpvAxyZ1vp3mpkN1/M87FVx/ZinjirJ47sODrNh0hOG56bx3xyfibvWLCF/uUAqZPmoIAOdOLubcycVdPY0JxTlMKI7nOonB1WtCFxEn8AjwSeAA8IGILDPGbG7dxxjzHzH7fwOYPQixqn4IRUsu6b3sqaC15KIpva+eXLWX7DQXdy8+ndx0d5f7OB2dk+pls0pYPHMkIlZn9LcvnMQjb+7kf96p4OnV+5g7Jp+5pfnsPNrIjmONLPljOd/65CSqmwKkuRy4ncLYgiy+/ed1+ENh1u6vZWiWh9V7rFr5tQvGDErnY7KKp4W+ANhpjNkNICJLgcuAzd3sfy1w18CEp06UttD7RtCKS18ZY1i5rZJzJhZ2m8x70ppwRYTC7DTuuOQ0huems+1oAz+8bFq0pX+03ssnH/xf/vvlrXicDvyhtusFCrI83HbJFL7z7Ho8Lgdzx+azZm8N50/pukWdquJJ6CXA/pjlA0CXV4uIyFigDPhHN9uXAEsAxowZ06dAVf+0JnSXR1vo8bBaiomOwl62HmngSL2Xcyd3LoX0h8fl4N8/Pr7T+mG56Tz91UXUewOcMa4AY6Cqyc+Dr23nUzNGcNaEQiYPy6G0IIucdBerK6pZWDZ0QGKyi3gSelffV7o75a8BnjXGhLraaIz5DfAbsIYtxhWhOiEmGEno2kKPi3VhkZ6affHeriqALmvbA21ayZDoYxEoyknjx1dMj66bObqt627RuMEZSZLM4hk7dgAYHbM8CjjUzb7XAM+caFBq4IRbW+hp2kKPRxdlXtWLBm8QgOIcPccSLZ6E/gEwUUTKRMSDlbSXddxJRCYD+cB7AxuiOhHRFrqWXOIiaKdoX/lD1ljzrjo91cnVa0I3xgSxrhZZDmwB/myM2SQiPxSR2Gn4rgWWGr1uOqm0JnS3JvS4iGinaF8FQgZPF2O71ckX1zh0Y8zLwMsd1n2/w/LdAxeWGijRhJ6WkeBI7MEhouPQ+8gfDON2aus8Geif1RRnQn4A3B7tFI2Ldor2mT8UxuNK3I1SVBtN6KkuaCV0j5Zc4mJd+p/oKOwlEAzj0RZ6UtCEnupCPvzGmdBbzdmJllz6zmqhaypJBvoppDgJ+QnQ96v3TlU6Dr3vAqFwlxNeqZNPP4UUJyE/ftGEHi+99L/vrE5RTSXJQD+FFCchP0GdJTluVslFM3pf+ENGSy5JQj+FFCdhPwFtocdPIKz3iO4TfzCk49CThH4KKc4R8hPUhB43xyk01epACWgLPWnop5DiHGE/Qe0UjZugnaJ9ZXWK6h/CZKAJPcU5wgGCDk+iw7ANvfS/77RTNHnop5DinCZASLRTNF7aKdp3Og49eeinkOKcYT8hbaHHzRqHnugo7MUfDGunaJLQTyHFuUyAkHaK9oHesaivAtpCTxr6KaQ4ZzhAWFvocXPoZC59pjX05KGfQopzESDs1IQeLy259J0OW0we+imkOJcJEHZoySVegqD3aOkbv87lkjT0U0hxbhPAaMklbg7RgktfGGMinaI6Dj0ZaEJPcW4CGC25xE1ECGvNJW7ByHulJZfkoJ9CinObIMapdyuKl2gLvU/8QWviGy25JAf9FJJQOBRi7evPEAz4e9xv3T+WsurpH/W4j7bQ+0Z02GKfBEJWQtcWenLQT2EQHdi5kfJlj3a7/YPnf8Hm917ptH710nuY9fbXWPvq//R4/BFv3cGi7fez+rmfAbB/xzqOH9kX3R4OhfBICDShx8269F8zery0hZ5c9FMYYBve+hv1d4+kvraKgj+ex7wPb8N0mI916wevs2vDKuav/S5Tl1/Dvu1rATDhMKuffZBFOx4AIOxv6fG16pxDARi6+Q+EQyHSn7qM/X/4WnS73++1Hrg0ocdLO0X7xt/aQteEnhT0UxhgTfvWkksTVYd2kyFWyaSluaHdPlP+/m+Mf+6i6PLh5T8FYNuHb7Jg4w+i68MtNT2+VmbYOm5WqJ7tH62kiBomNn1IwO8DiP4vLq2hx0tEdLbFPgiEtFM0meinMMBMczUALfVV0XUbf3cDh34wid0b32db+T86PSezcS8ANRtWALCq9MZ2xwIIBYOEgkG2vr+CvVvWAJAbrgcg2zRS89Ey67G0sPOjlVYMjXUAiDtzwH6+VKe3oIvfu7uOc979KwEtuSSLuD4FEblYRLaJyE4Rub2bfT4jIptFZJOIPD2wYdqHw1sLQEv1wei6BTV/Z6Q5yrhnL2TyS59ut/9O53gK/IcAyD38Lruc41h0/X9znDyc3hpMOMx7v7mZxv8aw5b7zmfKK1dRtPQSGuqqyZEWgsZBjrRQdOSf7HKWETJC7cbXADi2ez0AWSMmnYwfPSWIiJZc4nTXC5uij7WFnhx6/RRExAk8AlwCTAWuFZGpHfaZCNwBnGWMOR345iDEmlRCwSDv/fYWjuzf2W6922cl9MC+Nb0eI2yE40WLKA5XsmPtP5no20xl4UIAGh05uHy1rF/5Z8449AQeE2Caby17HKVkio+tbz4DwEFnCQCjg/uoyjmNXe6J5B95xzrGgY0ADJ84Z2B+6FOAdop2LRgK818vbeaSn/+T7UetUl9Nc9soLL3BRXKI58/qAmCnMWa3McYPLAUu67DPV4FHjDE1AMaYYwMbZvKp2FLOGQd/T8X//rHd+rSAldBzq9f3eoyjUohj2BRcEmbi85+iVnIZ9cmbAGh2DiEtUEvLAasVdPCK51k16VaKvrkSv3GRtvVvAFSnj7ZeVwKEsoqpGnYWE/xbaairxlG5hRpyKCguGagfO+UlQ8llzd5q5v3X69Q29zxs9WTaWdnIY2/vYcvhep5cZZUIq5va4tNO0eQQz6dQAuyPWT4QWRdrEjBJRN4RkVUicnFXBxKRJSJSLiLllZWV/Ys4gSq2lFNXcxyA2v1WopXq3e32yQxade1S3/Zej1eVVkL28InR5epLfs2oCdMA8LmHkBmsR+oPUk8mE2aexaLrvkd2bj470k5nhvcDa7/csujzJbuY3KmfxCVhdq1+ldyGXRzylCIO/WWLlyMJOkW3HG7geKOP/dU9j3Lqr3/uqOTrT60hFMcVsa3DEluTd2F2Gi+uO0Rdc6DdJGZackkO8XwKXX2X6ngmuICJwLnAtcBjIpLX6UnG/MYYM88YM6+oqKivsSbUql99jdI/nc/WJ78NgP+olbCzmva12y8r0lGZKb4ej7c+fT7e6Z+lcMyU6LrJ8y+IPg6k5ZEdrsfTfJgqR/v3qn7EGdHHUjA++tidO4wJc8+jxXjw7niTEv8eGnMnouKXDFeKNniDANS2dG6hP/zGDm57tv23v688Uc6Tq/ZypM5LMBTu9JztRxv4zKPvcazBGsb645e38vKGI7y3q4ryimpe3nC4yzgue+QdJn3vFZ5ZvY/a5gAAXz67jJrmAN9Y+lG7fbVTNDnE8ykcAEbHLI8CDnWxzwvGmIAxZg+wDSvBp4SGumrmHfkTAMNrygFw1+wCoNDX1vlpwmGGmLYhinVkRR+vKr6aTRc+E10ecvm9zPvUEgqHj4luj21Jh9LzyTWN5PqOUJ82rF082eMXRR9njWzr8EzPG0FaeiZ7PRMoO/YGOdICw6b1/wc/BYkk/krRBq+VPFuTKFgt5UZfkLd2VPLalqPR9d5AiNe3HOWdncdZ9OM3uPBnb3Vqef/pg/2srqjmyVVW4+O0EbkAPL16L1c++h5ff+rDTv0G/mCYdfut8uE7O49HW+ifnl3C9WeW8tb2SnLS225tqC305BDPp/ABMFFEykTEA1wDLOuwz/PAeQAiUohVgtlNitj1wXJcEmazexoloUM0N9aR22zVEYvNcbwtTYA13jxN2n4JD7rbyiEmLZfsgpHR5ez8YgAcTifhO6tZ+LX2V5RKxlDSJMDI4AG8mSPabRsz/Zzo46Kxbf3T2QXWfnV5UxmGNWyy+PSP9f8HPwW1fh1NZMdoWwu97Vz6+RvbWfyLtzlS76W6yR9N+nurmgHYdsRqSOyubOLZNfs5Vu/llqUfUXG8ieWbjgDw9Pt78QfDBCMXur284Uj0+Puqm9vFEFu/33yoPrqcn+Xmrn+dyj+/cx7l32v7Rqkt9OTQ66dgjAkCNwHLgS3An40xm0TkhyKyOLLbcqBKRDYDbwK3GmOquj6ivbz3u28z623r6kvv3K/ikjAVG95lRGA/NeTgEMPRiq0A1Ne07xdoyG0rh0h6LnlFbV0PQ4YWRx87nM5OdW5HVgEAGeInnDOy3bYh+YXRxwXD2r48tR7fUTLben2TwdjJOsKlLySS0RPZSm9N1nUxSXXbkQZ2VzZxuNYqm7Qm4D3HG63/q5qi+360r5Y7/rqBF9Ye4oanPuRATQsfm1TE8UY/W4/U0+gNMr4oi9sunsIFp1nn4YaDde1iqIl8O5g8LIc9VU0crG0hy+MkzeVERBg9NJM0lzO6f5q20JNCXJ+CMeZlY8wkY8x4Y8w9kXXfN8Ysizw2xphvGWOmGmOmG2OWDmbQJ0v5sl9xxv7HANiYNosxs84HYOrya8iRFrYXXgjAkfXWuO/6Sqv84jXWDSXMkLHRx470XHLzCvAbJ/Vk4nL3fDm+O6ctabvyx3Tavskz3drm9tBoMvAbJ7n5Vq29cOJ8APZknI7D6ez0XNU9RySjJyKfH6v3cuPTVgKG9iWXQ5FE3jpd7f5oQrf+b/0DlO52sHzTEd7YeowxQzPZcriesQWZfOuTVmlu+9FGGn1BCrPTuOHc8Tzy2Tm4ncKGA+0TemuJ5awJhRgD7+6qIj+r8zmbGym7uHTYYlLQP6s9KFj7KDud49n56Zcp/sLvKRw+mlXDruXDrHMon3cfc5b8is2e6czefC/v/+L/MGmZ9YXliNMqfbgKxtAsGdbjzCGIw0GtDKFBcnp97WET50UfZxaO7bR90n++gfc2qyujQbKplSHRVv7oSbM5SgEtpRd0ep7qWWtaSsRIl+89v5G/rz9M+V5ryofDdV4+3Gc9PlLvbbdvxxZ6q7MnFEZb149+bi7Xn1nKk19eyLSRuXicDnYca6DRFyQ7zUrEaS4nU4bn8tG+Wv7wXgU/e3079d5AtMRy9kTrm+LeqmbyMzsn9HMmWo0Il46kSgqu3ndJfZtXvUrLO48y65Zncbqst6Su6ihl4QreK72BM2aeFd130Q3ta92jvv4C+x7+JAuP/zW6rjqzjNLGfWQXl9EiGWDqcWdZg37qnfmEpfdW88iyKVRc/QaH332GufPO77Td7UnDjTVHS7MzB4PQWsRxuT0Uf38nxZ2epXpzsksudc0BrvjVO9xw7gTe29W+Svn3DYd5ddMRVt1xfrsx3wAbDtbT5Auy53hTu/Vnji/k9S3HGD00g6kjc7l78enRbeOKsthxtJEmX5DsmA7Ni6cN577l21hdYU018Yf39nLGOCuRnzYil/xMNzXNgS5b6A98ZiZfOruMohydLygZ6J9VoKH8T8xteJNd698GYGv5G+z8/b8DMGTKuT0+NzevgPwly3hv9Fc59tW1bLrwGfwjrZJHQckEvA5rHpW07HwAaqZ/ibppX4wrrtLT5nHGlx/Ak5be436Voy+hcuy/tFsnDoeOP+8HiZZcTk5G/2h/Dbsqm/jPv6yjwRfstD0UNtHRJq2yPE5eXHeIb/15LXuON1MQSbT5mW5OH2mNYFlYVtDpWBOKs6Mt9Ky0toT+5bPLKC3IZOSQdJYuWYTbKfw9MpQxP9PD6SOHRI/fUbrbydyx+f386dVA0xY6kFO/A4Cq9a9iZn2M0hevJj0yWmXczHN6eioABcNGccaX7weguKSMuslz+XDoaOaMLOW4IwNCkBZpoc+//KYBj3/R9f894Mc8VQ1EC72uJUB1k5+ywqxe9910yLpu4fOLxiJitY47Wn+gLaGLwI8un8a3/ryOldsq8QXDXHT6MJZvOkpxTjpTR+ZSnJPGxacP73ScScNyeGm9lahzYhJ6utvJ375+FmFjKMhOY86YfF7ZeIRMj5N0t5OpI3N5e+fxLksuKrmc8gndhMOU+K0RlrmH3mbL+8uZKgGCxsGawsUszOj9l7KjIQXDmHPx9QAEnFkQgKwcbcXYQbRT9AQS+mW/eJuKqmYqfvIvPe73+uajvL3jOKPyM/jR5db1Am/vOM7uDmWUdZEOy4nF2dQ0B7hizigO1bZw/wrr4rb5pUNZvukoRTlp5KS7Wf3drvtOxgxtm3UztoUOtCunjCuyzvnWOntrq18TevI75RP68SP7KKKJWrKZ6NvMhnd/Q6PJwHHrNhZmDznh4wdd1i9R1pChJ3wsNfgGolO0IjI23B8M43YKf1y1l5x0F5+ePSq6T1Wjj6/8wbpI7aLT2y4cK8xJ65TQW4cU/vCyadFa+rii7Oj2+aXWuVXcSx17WG5b6S47rftf/bJC69iNkRJQa8mlIFsTerI75RP64e1rKAK2lFzJGQd/z+z6N1mbcw5zBiCZAwRdWQSNg/R+tPTVyRctufTz+d5AKPq4qsnHyxuO8KOXNpPudnDWhEKKc6ykuu1o2xXFI/Myoo+76lysbvJTmO3hjPFtdfHWVrQITB6eQ1lhFqeX9HzODh8SX0JvPXaz3/pZJhRn88h1czhnUmG3z1HJ4ZTvNWs+sAGAcZfcjM+4cYghWPaJATt+KH8cB5wl2kFpE20ll/6l9C2H66OPjzf4Wbb2IOMKswiEDI/9c0902/bIlZ03njeeGz7edgFaUbaV0K+YU0K620FepCOy9XL9VqUFWYjAyCEZpLudvPGtj/Ols0p7jG14bAs9vYeE3kXt/19mjCA3vXOnqEoup3yWcVZu4Th5DBs1nu3pVh1z9LxLB+z4Cz/3Q0beVj5gx1MnRxwTEXZpfcwFOscbfeyrbmbR+AJmjhrCxpirMbcdbSQv081/XjiZ4phE29pC/9eZI9n6o0uide+OCT3d7WTM0Mxoa9rhkOgIne5keNqGy3asocfKi9TKY0tByh5O+ZJLXuNODqeNoxAILfga729ZwcKxkwfs+A6nE49erWkbcoI1l2XrDkVukgF7jjdR0xxgzNBMjtX7OFjbNh3u9qMNTBqW0ykJtyb01lEojZF5XaYM73wx2v1XzeyxdNKT3p638QcX6eX8NnRKf2KhYJDRwb005VmXRc86/xoW3vR4gqNSiapWA+8AAB4gSURBVOSI5vPeM7oxJnoJPsCH+2pYs7eGWy+aHF0Ga3RJfqabmiY/X39qDXf8dT1bD9czaVh2p2POHZvP6SNzGR/p9KxstKZh7thCB6sztKv18egtoWenuXTCLRs6JT8xv8+6jPpQxRbSJYBj+Om9PEOdKtpGufS+7983HOace9/knZ3WTU9+98895KS7+OIZpWR6nHy0zxo/PmZoJkOzPFQ3+3l5wxGeWb0fp0O4Zn7nOXrGF2Xz95vPiQ4j/NyisdH1A6mnGrqyr1MuoX+04kk8Px7Grg2rOLbdqm3njZ2Z4KhUspA+dIruPGbNo/La5qPsr27mlY2HuW7hGLLSXBRmp0VLLKOHZpKX6Yne/eeMcQUsu+lspvUyKgXg1gsns/VHFw/YfONDI38osjxaBkxFp9Sfab/Py+nv3AwCx7e8RcbuVzlOHmOnzk90aCpJOPpQQm/N+bsqG3lx/SHCBq4/sxSAwmwP+6qbGZLhZkiGm6FZbSNErls4htI4riIFq7Mz3TFwyffpry7kbx8eZEiGjlhJRadUC33z28/jEWtsbfqe15jhXcOOss+Slp7ZyzPVKSPSQo/nwqJjDVZ9e+2+WjYdrKckL4MRQ6wx5a13DZpfal0hnBdzlWVvFwANpinDc7nj0tN6HRGj7OmUaqH7tiyn2aRR6xjCzJbVAEy86IYER6WSiSN6y6Le962MJPQGX5A3th7lrPFtF95cu2AMxbnHeOAzVjlvaMyl9bFXbCo1kE6JhN7UUEvFw59ioX8D6zIXYhBGthzjgAxn1PDRvR9AnTKE1hZ67/tWNng5fWQumw/X4w2EmRwztPCaBWO4ZkFbp2fsTIXFuTrVrBocp0TJZcfqVzndb10R6i09H2+ONXLgSI7eQFm1J3EOW6xrDlDZ4GPK8FymRzo3p/QwhLB1YqvsNBeZnlOiHaUS4JRI6N5t/8BvXKz/+O+Ye8V/IAXjAAiNnNfLM9WpxtHL9LneQIhVu6uY9aMVHKrzUpybxrmTrLv2dHXxT6vWTkhtnavBlLJNhebGOtyedNyeNIYdf5/t6dOYcd6VAORPPAP/ZifDZ16Y4ChVsmkrubTP6IFQmKsefY+1+2vJSXdFE35RdhqXzy6hODedicXdjxV3OR0MyXAntENUpb6UTejHHjybwyUX4Ww4yIJwBatGfzO6bdKcj+Odso+xmQN7sYZKAd200Gua/KzdX8vkYTntZkrMz3IzNMsTvQCoJ6WFWUws7v1+skr1V8om9OLQUWqqNjO7+R3Kcy9gzlV3tNuerslcdcHRzXA+X+SioK+cU8aZEwopr6jmlqVrGTM0/mmRn/zyAr2cXg2qlEzo4VCITPGR5z1oLY85q9f7cioF3d/gwhe0rl9IczspycugZFYJi8YV9GkIYo5OP6sGWVzNBRG5WES2ichOEbm9i+3Xi0iliKyN/PvKwIcaP5/XmjBpWMi6f6IrS2//puLT3T1FvQGrhR47A6GOJ1fJptcWuog4gUeATwIHgA9EZJkxZnOHXf9kjBn4OyD3Q0tTPRlAplgXfnhyOt8BXamuRG9w0WF9aws93a1zoKjkFU8LfQGw0xiz2xjjB5YClw1uWCfG29zYbjldE7qKU2sLvVPJpYsWulLJJp6zswTYH7N8ILKuo38TkfUi8qyIdHn5pYgsEZFyESmvrKzsR7jx8bc0tFvOytN7Iar4tM222H59a6eoJnSVzOI5O7vq9u/4jfRFoNQYMwN4HXiiqwMZY35jjJlnjJlXVFTUt0j7wN/S/q7p2XmD91oqtUSncumQ0Vtv/pzm0pKLSl7xJPQDQGyLexRwKHYHY0yVMcYXWfwtMHdgwuufQEtbySVoHGTn5CUwGmUn3d2BrrWFnu7WFrpKXvGcnR8AE0WkTEQ8wDXAstgdRGREzOJiYMvAhdh3QV9byaVBshGH/hKq+Di6Lbm0DVtUKln1OsrFGBMUkZuA5YATeNwYs0lEfgiUG2OWATeLyGIgCFQD1w9izL0Kettuxtso2eigRRWv7sehaw1dJb+4LiwyxrwMvNxh3fdjHt8B3NHxeYkS8rWVXJqdeqm1il/349B12KJKfinZ3DD+tk5Rn6t/d0VXp6boKBd02KKyn5Q7O48d3EO49mB02e/p/Ua8SrVqG+XSti4cNniDIRwCLofeuk0lr5Sby6X4t7MojlkOebSFruLXcRy6MYZz7n2TmmY/aS6n3otTJbWUS+itWoyH9YWXkjHtU4kORdmIo8Mdi5r8IQ7WWp3ssbeRUyoZpWxC90o6C7/R5fVNSnWr7dJ/6/+aJn90m15UpJJdStXQw6FQ9HEgdf9WqUHUVnKxMnpNc0xC14uKVJJLqTO0saE2+jjDtPSwp1JdaxuHbv1f0xyIbtMRLirZpdQZ2txQE32cI5rQVd+1dXpaGb02poWuY9BVskuphN4Sk9CV6g9HhwuLqtvV0FPq10WloJQ6Q32Ntb3vpFQPJFJ06brkoi10ldxSKqH7m+oSHYKyubZL/zuXXHQIukp2KZXQA81tLfQNabMTGImyq47T58a20Fsn6FIqWaXU2L5QSz0AB7/wLlPHTE5wNMqO2kounVvomtBVskupFnrYa5VccgtG4HSl1N8qdZJ0GORCdZM/On+LLxDq+klKJYmUSuh4GwgbIStbJ+RS/RO9wUVkubY5wOihmQD4tYWuklxqJXRfPU2k43DqaATVP22X/hvCYcPxRh9lhVlA25zoSiWrlErozkAjTZKV6DCUjcWOQz/W4MMXDDOtxPrG59UWukpyKZXQXYEGvI7MRIehbK2tU7SiyrpRyvRIQr942vCERaVUPFKq59AdbMTrzE50GMrGYoct7o0k9CnDcyj/3gXkZej0uSq5pVRCzwrUUpemrSjVf46YjF5R1YzbKYwYko7LmVJfZlWKSpmzNBwKMTx0CG9uWaJDUTbWNtuiYW9VE6PzMzWZK9tImTP16IGdZIgfKZyY6FCUjUlMp2jF8WbGFmifjLKPlEnolXs2ApAzamqCI1F2FjsOvbLRx7Dc9MQGpFQfpExCbz68FYDismkJjkSlgrAx+AIhnTJX2UpcZ6uIXCwi20Rkp4jc3sN+V4qIEZF5AxdifKRqJ/VkUlBccrJfWqWQ2JKLPxTGowld2UivZ6uIOIFHgEuAqcC1ItKpriEiOcDNwPsDHWRvQsEgxdVrOOQaizj0F1D1nyPmnqL+oCZ0ZS/xnK0LgJ3GmN3GGD+wFLisi/1+BNwLeAcwvrisefFXlIX30jjrSyf7pVWKaW2hB8OGsAGPTiOhbCSehF4C7I9ZPhBZFyUis4HRxpiXejqQiCwRkXIRKa+srOxzsN3J2PocFY7RzL30KwN2THVqam2ht87bkubWFrqyj3jO1q7u02KiG0UcwE+Bb/d2IGPMb4wx84wx84qKiuKPsheFvgMcz56s5RZ1wlpP9ta5zz06Bl3ZSDxn6wFgdMzyKOBQzHIOMA1YKSIVwCJg2cnqGPW2NDHMHCcwRC8oUieuteQSTehaQ1c2Es/Z+gEwUUTKRMQDXAMsa91ojKkzxhQaY0qNMaXAKmCxMaZ8UCLu4OjebTjE4C6acDJeTqU46VBy0YSu7KTXs9UYEwRuApYDW4A/G2M2icgPRWTxYAfYm5oD2wDIGTkpwZGoVNCx5KLj0JWdxDU5lzHmZeDlDuu+382+5554WPHzHt0OwLBSvUJUnbjWFnrr7ea0hq7sxPZnq1Tvpo4s8gp1lkV14lpvcKElF2VHtj9b3b5qah35iQ5DpQiJFF20U1TZke3PVmewBb/oBEpqYEjHFrqWXJSN2P5sdYW9BJwZiQ5DpQgdtqjszPZnqyfsJejQFroaGNFOUU3oyoZsf7a6w16C2kJXA6Rjp2iaS+dyUfZh+4SeFvYScmkLXQ2Mjp2iOg5d2Yntz9Y0fIRdepswNTDaaug6bFHZj+3P1nTjw7i05KIGRtsoF52cS9mPrc9WEw6TKT5wawtdDYy2kou20JX92Pps9bY0AWA8WQmORKWKaMkloKNclP3Y+mxtaaoHQNxaclEDI/YGFyLgcnR1OwClkpOtE7qvpREAZ5q20NXAaE3f3mAYj9MRHZeulB3YOqH7m62E7tCErgZIawtdbxCt7MjWZ6zfa9XQtYWuBkxMg1zHoCu7sfUZ64+UXFzpmtDVwJB2CV2vElX2YuuEHvRZLXRN6GqgOGIyupZclN3Y+owNea0WelpGdoIjUakitgtULypSdmPrMzbkawbAowldDZDYkou20JXd2PqMDfutkktaZk6CI1GpQksuys5sfcaaSEJP14SuBoGWXJTd2PqMNf4WADI0oasBoiUXZWf2PmMDTfiMG6fLlehIVIrQkouys7jOWBG5WES2ichOEbm9i+1fE5ENIrJWRN4WkakDH2pnjkALXvGcjJdSp4h2o1w0oSub6fWMFREn8AhwCTAVuLaLhP20MWa6MWYWcC/w4IBH2gWnr5ZG0REuauDEttDTtIaubCaeM3YBsNMYs9sY4weWApfF7mCMqY9ZzALMwIXYvUzvUWrdxSfjpdQpIraG7taErmwmnjO2BNgfs3wgsq4dEblRRHZhtdBv7upAIrJERMpFpLyysrI/8baTF6ykJX3YCR9HqVaxsyu6XTrTorKXeBJ6V2d1pxa4MeYRY8x44Dbge10dyBjzG2PMPGPMvKKior5F2kE4FKIwXEUga+QJHUepjlpzurbQld3Ec8YeAEbHLI8CDvWw/1Lg8hMJKh41xw/jkSCOIZrQ1cBqbcHoOHRlN/GcsR8AE0WkTEQ8wDXAstgdRGRizOK/ADsGLsSu1RypAMAzdHTPOyrVR61lF22hK7vpdQC3MSYoIjcBywEn8LgxZpOI/BAoN8YsA24SkQuAAFADfHEwgwZoPLYPgOyiMYP9UuoUY4xVUdSEruwmrityjDEvAy93WPf9mMe3DHBcvfJVW/20+SNKT/ZLqxQXjvQQaaeoshvbNkHCdQcJGCdDizoNuFFqQGgNXdmNbc9Yh6+ORsnC4dS7yqjBoSUXZTe2PWMdwRZaJD3RYagUppf+K7ux7RnrDDbjk4xEh6FSmLbQld3Y9ox1hZrxO7SFrgaP26mdospebJvQPaEWAk5toavBo52iym5se8a6wy0EnJmJDkOlMC25KLux7RmbFm4h5NKErgaPWztFlc3Y9oxNN15N6GpQaQ1d2Y1tE3qmacG4sxIdhkphWkNXdmPLMzYcCpEpPoxHE7oaPFpDV3ZjyzPW29IIgGhCV4NIE7qyG1uesc2N1h3vNKGrweTRybmUzdgyofuarYTuSNMbRKvBoy10ZTe2PGO9TQ0AuDI0oavBowld2Y0tz9hASyShp+ckOBKVyjShK7ux5RnbmtDd6dpCV4NHZ1tUdmPLMzbojST0zNwER6JSmY5DV3ZjyzM2GBm2mJ6lJRc1ePRKUWU3tkzoYX8TAGnaQleDyOnQhK7sJa6bRCcb47Na6BnaQleDSMT+CT0QCHDgwAG8Xm+iQ1F9lJ6ezqhRo3C73XE/x54JPdJCz8jUhK5UTw4cOEBOTg6lpaUp8QfqVGGMoaqqigMHDlBWVhb382xZcpFAC17j1htEK9ULr9dLQUGBJnObEREKCgr6/M3Kngk96MUnnkSHoZQtaDK3p/58bnEldBG5WES2ichOEbm9i+3fEpHNIrJeRN4QkbF9jqQPJOTFR9pgvoRSStlOrwldRJzAI8AlwFTgWhGZ2mG3j4B5xpgZwLPAvQMdaCxHyIdfW+hKKdVOPC30BcBOY8xuY4wfWApcFruDMeZNY0xzZHEVMGpgw2zPGfISEG2hK2UHDz30EKeddhr5+fn85Cc/6dcxamtr+eUvfznAkXXt3HPPpby8vNvt2dkDe4V6b6/XF/GMcikB9scsHwAW9rD/l4FXutogIkuAJQBjxoyJM8TOnCEfAYcmdKX64gcvbmLzofoBPebUkbnc9a+n97jPL3/5S1555ZU+jdboqDWhf/3rX+/3MZJBMBjE5Rq8wYXxtNC7qsybLncU+RwwD7ivq+3GmN8YY+YZY+YVFRXFH2UHzrCXoCZ0pZLe1772NXbv3s3ixYv56U9/yk033QTA9ddfz80338yZZ57JuHHjePbZZ6PPue+++5g/fz4zZszgrrvuAuD2229n165dzJo1i1tvvZWVK1fyqU99Kvqcm266id///vcAlJaWctdddzFnzhymT5/O1q1bAWhqauJLX/oS8+fPZ/bs2bzwwgsAtLS0cM011zBjxgyuvvpqWlpaev25vvvd7zJz5kwWLVrE0aNHAXjxxRdZuHAhs2fP5oILLoiuv/vuu1myZAkXXnghX/jCF/r1evGK50/FAWB0zPIo4FDHnUTkAuC7wMeNMb6BCa9r7rAPr0uvElWqL3prSQ+GRx99lFdffZU333yTl156qd22w4cP8/bbb7N161YWL17MlVdeyYoVK9ixYwerV6/GGMPixYt56623+MlPfsLGjRtZu3YtACtXruzxdQsLC/nwww/55S9/yf33389jjz3GPffcwyc+8Qkef/xxamtrWbBgARdccAG//vWvyczMZP369axfv545c+b0eOympiYWLVrEPffcw3e+8x1++9vf8r3vfY+zzz6bVatWISI89thj3HvvvTzwwAMArFmzhrfffpuMjAwefPDBPr1eX8ST0D8AJopIGXAQuAa4LnYHEZkN/Bq42BhzbMCi64Y77KfJoZ2iStnZ5ZdfjsPhYOrUqdHW7IoVK1ixYgWzZ88GoLGxkR07dvS5RHvFFVcAMHfuXP76179Gj71s2TLuv/9+wBqjv2/fPt566y1uvvlmAGbMmMGMGTN6PLbH44l+O5g7dy6vvfYaYF3EdfXVV3P48GH8fn+7EtPixYvJyMgA6PPr9UWvCd0YExSRm4DlgBN43BizSUR+CJQbY5ZhlViygb9Exk7uM8YsHrAoO3AbH2Fn+mAdXil1EqSltZVNjTHR/++44w7+/d//vd2+FRUV7ZZdLhfhcDi63PECnNZjO51OgsFg9NjPPfcckydP7hRLX8Z8u93u6P6xx//GN77Bt771LRYvXszKlSu5++67o8/Jymp/u8zBujYgrnHoxpiXjTGTjDHjjTH3RNZ9P5LMMcZcYIwZZoyZFfk3aMkcwGN8hDShq0GydMkilt10VqLDOCVddNFFPP744zQ2WvM1HTx4kGPHjpGTk0NDQ0N0v7Fjx7J582Z8Ph91dXW88cYbcR374Ycfjv7x+OijjwD42Mc+xlNPPQXAxo0bWb9+fb9ir6uro6SkBIAnnnii2/0G6vW6YssrRT34MS5N6GpwLBpXwIxReYkO45R04YUXct1113HGGWcwffp0rrzyShoaGigoKOCss85i2rRp3HrrrYwePZrPfOYzzJgxg89+9rPREk1P7rzzTgKBADNmzGDatGnceeedANxwww00NjYyY8YM7r33XhYsWNCv2O+++26uuuoqzjnnHAoLC7vdb6BeryvS+tfqZJs3b57p79jLlruKWDf831h0w6MDHJVSqWXLli2cdtppiQ5D9VNXn5+IrDHGzOtqf9u10E04TIb4Me6MRIeilFJJxXbT5/p8LaQD4tKErpQaPAsXLsTnaz8C+49//CPTp09PUES9s19C91oJHbfW0JVSg+f9999PdAh9ZruSS6DFurmFaMlFKaXasV1C93mtOcAcHk3oSikVy3YJPeCzWuia0JVSqj37JfRIC93pyUxwJEqp3vR32ttLL72U2traAYlhoKe7TWb2S+g+TehK2UV3CT0UCvX4vJdffpm8PL24q69sN8olFEno7jQtuSjVJ6/cDkc2DOwxh0+HS7q/aUXstLdut5vs7GxGjBjB2rVr2bx5M5dffjn79+/H6/Vyyy23sGTJEsCaAre8vJzGxkYuueQSzj77bN59911KSkp44YUXyMjIYNeuXdx4441UVlaSmZnJb3/7W6ZMmcKePXu47rrrCAaDXHzxxQP78yY527XQg5GE7krXFrpSye4nP/kJ48ePZ+3atdx3332sXr2ae+65h82bNwPw+OOPs2bNGsrLy3nooYeoqqrqdIwdO3Zw4403smnTJvLy8njuuecAWLJkCQ8//DBr1qzh/vvvj9784pZbbuGGG27ggw8+YPjw4Sfvh00C9muhB6zJ4N1pmtCV6pMeWtIny4IFC9pNK/vQQw/xt7/9DYD9+/ezY8cOCgoK2j2nrKyMWbNmAdZ0tRUVFTQ2NvLuu+9y1VVXRfdrvQjonXfeiSb9z3/+89x2222D+jMlE9sl9LBfE7pSdhU7jezKlSt5/fXXee+998jMzOTcc8/tNA0utJ9m1+l00tLSQjgcJi8vL3rDi44Ga3raZGe7kouJJHRPhiZ0pZJdx2lvY9XV1ZGfn09mZiZbt25l1apVcR83NzeXsrIy/vKXvwDWXOfr1q0D4KyzzmLp0qUA0WlqTxX2S+iRkosnPauXPZVSidZx2ttYF198McFgkBkzZnDnnXeyaNGiPh37qaee4ne/+x0zZ87k9NNPj94j9Oc//zmPPPII8+fPp66ubsB+Fjuw3fS5H614ErPuT0y/5VncHr1RtFI90elz7a2v0+faroY++8LPwYWfS3QYSimVdGxXclFKKdU1TehKpbhElVXVienP56YJXakUlp6eTlVVlSZ1mzHGUFVVRXp63+77YLsaulIqfqNGjeLAgQNUVlYmOhTVR+np6YwaNapPz9GErlQKc7vd7a7MVKlNSy5KKZUiNKErpVSK0ISulFIpImFXiopIJbC3n08vBI4PYDgnm8afOHaOHTT+REuG+McaY4q62pCwhH4iRKS8u0tf7UDjTxw7xw4af6Ile/xaclFKqRShCV0ppVKEXRP6bxIdwAnS+BPHzrGDxp9oSR2/LWvoSimlOrNrC10ppVQHmtCVUipF2C6hi8jFIrJNRHaKyO2Jjqc3IlIhIhtEZK2IlEfWDRWR10RkR+T//ETH2UpEHheRYyKyMWZdl/GK5aHIZ7FeROYkLvJorF3Ff7eIHIx8BmtF5NKYbXdE4t8mIhclJupoLKNF5E0R2SIim0Tklsh6W7z/PcRvl/c/XURWi8i6SPw/iKwvE5H3I+//n0TEE1mfFlneGdlemsj4AWuaRrv8A5zALmAc4AHWAVMTHVcvMVcAhR3W3QvcHnl8O/D/Eh1nTGwfA+YAG3uLF7gUeAUQYBHwfpLGfzfwn13sOzVyDqUBZZFzy5nA2EcAcyKPc4DtkRht8f73EL9d3n8BsiOP3cD7kff1z8A1kfWPAjdEHn8deDTy+BrgT4l8/40xtmuhLwB2GmN2G2P8wFLgsgTH1B+XAU9EHj8BXJ7AWNoxxrwFVHdY3V28lwF/MJZVQJ6IjDg5kXatm/i7cxmw1BjjM8bsAXZinWMJYYw5bIz5MPK4AdgClGCT97+H+LuTbO+/McY0RhbdkX8G+ATwbGR9x/e/9XN5FjhfROQkhdsluyX0EmB/zPIBej5hkoEBVojIGhFZElk3zBhzGKxfAqA4YdHFp7t47fR53BQpSzweU+JK2vgjX99nY7USbff+d4gfbPL+i4hTRNYCx4DXsL411BpjgpFdYmOMxh/ZXgcUnNyI27NbQu/qr1+yj7s8yxgzB7gEuFFEPpbogAaQXT6PXwHjgVnAYeCByPqkjF9EsoHngG8aY+p72rWLdckYv23ef2NMyBgzCxiF9W3htK52i/yfdPHbLaEfAEbHLI8CDiUolrgYYw5F/j8G/A3rJDna+tU48v+xxEUYl+7itcXnYYw5GvlFDQO/pe1rfdLFLyJurGT4lDHmr5HVtnn/u4rfTu9/K2NMLbASq4aeJyKtNwOKjTEaf2T7EOIv9w0KuyX0D4CJkV5nD1ZHxLIEx9QtEckSkZzWx8CFwEasmL8Y2e2LwAuJiTBu3cW7DPhCZLTFIqCutTSQTDrUlT+N9RmAFf81kdEKZcBEYPXJjq9VpP76O2CLMebBmE22eP+7i99G73+RiORFHmcAF2D1A7wJXBnZreP73/q5XAn8w0R6SBMm0b2yff2H1bO/Hau29d1Ex9NLrOOwevHXAZta48Wqs70B7Ij8PzTRscbE/AzW1+IAVgvky93Fi/WV85HIZ7EBmJek8f8xEt96rF/CETH7fzcS/zbgkgTHfjbWV/b1wNrIv0vt8v73EL9d3v8ZwEeRODcC34+sH4f1h2Yn8BcgLbI+PbK8M7J9XCLjN8bopf9KKZUq7FZyUUop1Q1N6EoplSI0oSulVIrQhK6UUilCE7pSSqUITehKKZUiNKErpVSK+P/we1aQjEDQWgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title(\"Accuracy\")\n",
    "plt.plot(trained_logs[\"test_acc\"] + finetuned_logs_hard[\"test_acc\"], label=\"finetuned_hard\")\n",
    "plt.plot(trained_logs[\"test_acc\"], label=\"trained\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEICAYAAABRSj9aAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3dfZRU1Znv8e9T3dANtAoCyRjR0Cq+YCSILXGuudHRqCB3wIxoTG4iomsRTUxMMs4EV5aOmsyKRmdMNHiNRrIcY2J8SUaS4EsSw5hJCLFVRF50QEKk0SiCEEHemnruH3Wq+oVuep/qwnOqzu+zFlB16lTx9KlTT+969j57m7sjIiK1K5d0ACIism8p0YuI1DglehGRGqdELyJS45ToRURqnBK9iEiNU6IXEalxSvQi3ZjZGjP7aLdtp5pZ3sy2mNnbZvaSmc1MKkaROJToRcK96u5NwP7Al4C7zOyohGMS6ZMSvUhMXjAf2AiMSzoekb7UJx2ASLUxsxzwf4ARwKqEwxHpkxK9SLj3mdkmYBCFz86X3f25hGMS6ZNKNyLhXnX3oRRq9LcCpyUcj0gQJXqRmNx9B/AV4DgzOyfpeET6okQv0rMBZtZY/EO3Mqe77wT+DbgmkehEYjDNRy/SlZmtAd7fbfPvgNHuPqrTfoOBV4CZ7v6zdy9CkXiU6EVEapxKNyIiNU6JXkSkxinRi4jUOCV6EZEal7orY0eMGOGjR49OOgwRkaryzDPPvOnuI3t6LHWJfvTo0bS2tiYdhohIVTGzP/f2mEo3IiI1ToleRKTGKdGLiNQ4JXoRkRqnRC8iUuOCEr2ZTYoWQ15lZrN7ePwjZvasmbWb2fRuj80ws5XRnxmVClxERML0mejNrA6YA0wGxgKfMLOx3XZ7BbgI+GG35x4I/AvwIWAi8C9mNqz/YYuISKiQcfQTgVXuvhrAzO4HpgHLizu4+5rosXy3554F/NLdN0aP/xKYBPyo35F388a6P7H60dsq/bI168AJUzlywqlJh1FV/rxhKz95dh2a8bVvA+tzfOqk9zN08MCkQxHCEv3BwNpO99sotNBD9PTcg7vvZGazgFkAhx56aOBLd7Xp9VeYuHZuWc/Nmpw5z294AZToY/nholf47lOrMUs6knQr/h486IBBnHvCqL3vLO+KkETf02kd2qQJeq673wncCdDS0lJWc+nICafAhE3lPDVzXvr6RMx3Jx1G1WnPO00N9Sy97qykQ0m1trfe4cM3/obdeX3zSYuQztg24JBO90cBrwa+fn+eK/uMYSo/xJZ377HlIl1Z9JXHg9uDsq+FJPqngTFm1mxmA4ELgHmBr/84cKaZDYs6Yc+MtkmC3HKEfymTInd6/o4qXeSiY6S2RHr0mejdvR24nEKCXgE84O7LzOx6M5sKYGYnmlkbcB7wXTNbFj13I/A1Cr8sngauL3bMSnIK+UqfwnIoz/fNoqOkyk16BM1e6e7zgfndtl3T6fbTFMoyPT13LqBe0lRR6aYc7k4up1Tfl2JntUo36aErYzNIpZvy5F0t+hCm0k3qKNFnkEo35XG81NEovSuWbnS9QXoo0WeSqblVBleLPkhH6UbSQok+g9xyatGXwUEt+gC54vBKnWKpoUSfUUr08bm7rooNUDxEeWX61FCizyBX6aYsKt2EUWds+ijRZ5BKN+VxRy36AB1XxkpaKNFnkmF0n2hU+uJ4qf4sveto0SvVp4USfQZpxpbyaBx9GHXGpo8SfRaZYa4WfVyF0o1SfV/UGZs+SvSZZKrRl0GX9IfROPr0UaLPoMIUCBKbQ06Hrk8q3aSPTtsMcnXGlqUwH71KN6FUukkPJfqM0uyV8RWujE06ivTTMUofJfosspzapWXQBVNhOko3akykhRJ9Bql0Ux4HjaMP0DHqJtEwpBMl+kwyNCYiPleTPoipMzZ1lOizyNSlWA7l+TClNWPVmEgNJfoMUummPFp4JEzxGKl0kx5K9FlkWjO2HGrRx6RzLDWU6DPI9baXxV2dsaFypl6gNNEnPotMpZty5LXwSDAz0wVTKaJEn0ma66YcOmLhclrbJlWU6DPITYm+HJq9Mpxh6oxNESX6TFJnbHm8NHRQ+mAaXpkmSvRZpCkQypLXUoLBcromL1WU6DNI4+jL45q9MlihdKNMnxZK9BmlGn18mr0ynKkzNlWU6LNIC4+URZ2x4XKmpkSa6BOfRWbkVLqJrbDwiIQwtPBImijRZ5DSVfnUoA+k0k2qBCV6M5tkZi+Z2Sozm93D4w1m9uPo8UVmNjraPsDM7jGzF8xshZldVdnwpSyWU42+DJrrJpymikiXPhO9mdUBc4DJwFjgE2Y2tttulwBvufsRwC3AjdH284AGdz8OOAH4TPGXgCQrp0Qfm+NKYIHMVLpJk5AW/URglbuvdvedwP3AtG77TAPuiW4/BJxuhV4rB4aYWT0wCNgJ/LUikUv5LIcGOceXz6t0E8pQ6SZNQhL9wcDaTvfbom097uPu7cBmYDiFpL8VeA14BbjZ3Td2/w/MbJaZtZpZ6/r162P/EBKXpkAoh6Nx9KEKo250jqVFSKLv6czu/g72ts9EYDfwPqAZ+EczO2yPHd3vdPcWd28ZOXJkQEjSL2Yq3ZTBHRXpAxVKN0lHIUUhib4NOKTT/VHAq73tE5VpDgA2Ap8EHnP3Xe7+BvA7oKW/QUv/uEo3ZSksDp50FNXBzFS6SZGQRP80MMbMms1sIHABMK/bPvOAGdHt6cCT7u4UyjWnWcEQ4CTgxcqELuUzcvoUxueodBOoUKPXOZYWfSb6qOZ+OfA4sAJ4wN2Xmdn1ZjY12u1uYLiZrQK+DBSHYM4BmoClFH5hfN/dl1T4Z5C4NE1xWbTwSDhNgZAu9SE7uft8YH63bdd0ur2dwlDK7s/b0tN2SZoSfTk01004dcami66MzSJdMFUWd42jD1WYAiHpKKRIiT6jlOjj0xELp87YdFGizyC3OnUpliGv2SuDmVaYShUl+iwytPBIOTR7ZTB1xqaLEn0maZBgOdQZG84wDa9MESX6LFJnbFncNStjqJypTyNNlOizSFMglEULj4QzM426SREl+gwyjJzpUxhXYSnBpKOoDroyNl2U6DPIozVjPa8O2TgKaUuZPoQ6Y9NFiT6LomapWlzxFC6YSjqK6mC6MjZVlOizqNiiV6KPRaWbcDm16FNFiT7D8vndSYdQVbTwSDjDtJRgiijRZ5FKN2VRiz6cavTpokSfRaXSjTpj4ygsPKJMH6JQo5e0UKLPpKhFr1E3seTdNegmkIZXposSfRapM7Y8yvPBVLpJFyX6DLJSjV4t+jgKc90o1YfIqXSTKkr0WRQlq7xKN7FoHH04MzTqJkWU6DNJo27KkVfpJlihRp90FFKkRJ9FqtGXxXGVbgJp1E26KNFnkWnUTTk06CZcoTNWqT4tlOgzKUpX6oyNxbWUYDCVbtJFiT6LotKNOmPjU54PkzNNgZAmSvQZpOGV5dHCI+E0jj5dlOizSJ2xZdFcN+E0TXG6KNFnkVr0ZdHsleEMtJRgiijRZ1KUrPRJjMUdcvrEBDFDq4OniE7bLCqWbvRJjKXwe1Et+hA5lW5SRYk+g6w0BYIWHonHVaMPVJgCIekopEiJPou08EhZdMFUOMN0fqWIEn0GWVS60fi3eLTwSDgzlejTJCjRm9kkM3vJzFaZ2eweHm8wsx9Hjy8ys9GdHhtnZgvNbJmZvWBmjZULX8rhmr2yLHlX6SaUmal0kyJ9JnozqwPmAJOBscAnzGxst90uAd5y9yOAW4Abo+fWAz8ALnX3Y4FTgV0Vi17KVJwCQZ/EOFS6CWeg8ytFQlr0E4FV7r7a3XcC9wPTuu0zDbgnuv0QcLoVevzOBJa4+/MA7r7B3dUDmDDTBVNlcdfslaFyKt2kSkiiPxhY2+l+W7Stx33cvR3YDAwHjgTczB43s2fN7J97+g/MbJaZtZpZ6/r16+P+DBJXafZK/c6No7DCVNJRVAfTXDepEpLoezq1u7+Dve1TD3wY+L/Rvx8zs9P32NH9TndvcfeWkSNHBoQk/VGa60ZtrlgKpRtl+hCavTJd6gP2aQMO6XR/FPBqL/u0RXX5A4CN0fb/cvc3AcxsPjAB+HU/45b+KJZu1KKPxWu4M3bXrl20tbWxffv2irzezGMHsDtfz4oVKyryetKhsbGRUaNGMWDAgODnhCT6p4ExZtYMrAMuAD7ZbZ95wAxgITAdeNLd3cweB/7ZzAYDO4FTKHTWSpJK4+gTjqPKOLXbGdvW1sZ+++3H6NGjK9IPsebNrezcnefI9+5XgeikyN3ZsGEDbW1tNDc3Bz+vz9JNVHO/HHgcWAE84O7LzOx6M5sa7XY3MNzMVgFfBmZHz30L+HcKvywWA8+6+y9i/FyyD3SMo9fwyjgKc93UZqrfvn07w4cPr1hnc61+80mamTF8+PDY37xCWvS4+3xgfrdt13S6vR04r5fn/oDCEEtJC81eWZZan49eI4qqQznvk66MzSQNryyH5jSLSadXaijRZ5AVyw9q0cejUTexKM+nhxJ9BnXMXqmPYhyOU6Ml+tS49dZbOeaYYxg2bBg33HBDWa+xadMmbr/99gpH1rNTTz2V1tbWirzWiy++yPjx4zn++ON55plnKvozKNFnkKPO2HJoKcFw5R6m22+/nfnz5/PWW28xe/Ye02oFeTcTfSX953/+J9OmTeO5555j+PDhFf0ZgjpjpbaYpikuS6EztvYz/XU/W8byV//ar9fY0Z4nn3cGDawDYOz79udf/v7YvT7n0ksvZfXq1UydOpWLL76Yl19+me985ztcdNFF7L///rS2tvKXv/yFb37zm0yfPh2Am266iQceeIAdO3bwsY99jOuuu47Zs2fz8ssvM378eM444wymTJnCzTffzM9//nMALr/8clpaWrjooosYPXo0M2bM4Gc/+xm7du3iwQcf5Oijj2br1q18/vOf54UXXqC9vZ1rr72WadOmsW3bNmbOnMny5cs55phj2LZtW68/z+7du7nkkktobW3FzLj44ov50pe+xOLFi7n00kt55513OPzww5k7dy4LFy7kW9/6FnV1dTz11FO8973v7fIz3HTTTf16P5ToM6hjrhu16OPQFAjxxG1G3HHHHTz22GP85je/KSXlotdee43//u//5sUXX2Tq1KlMnz6dJ554gpUrV/LHP/4Rd2fq1Kk89dRT3HDDDSxdupTFixcDsGDBgr3+vyNGjODZZ5/l9ttv5+abb+Z73/se//qv/8ppp53G3Llz2bRpExMnTuSjH/0o3/3udxk8eDBLlixhyZIlTJgwodfXXbx4MevWrWPp0qVA4ZsGwIUXXshtt93GKaecwjXXXMN1113Ht771LS699FKampq48sorWbNmTZefob+U6LOomK3Uoo8lK7NX9tXyDvHKxnd4Z2c7R//N/hWICM455xxyuRxjx47l9ddfB+CJJ57giSee4Pjjjwdgy5YtrFy5kkMPPTTWa//DP/wDACeccAI/+clPSq89b948br75ZqBwncErr7zCU089xRe+8AUAxo0bx7hx43p93cMOO4zVq1fz+c9/nilTpnDmmWeyefNmNm3axCmnnALAjBkzOO+8HkemV5QSfQYVR92odBOueKw01jxMpY9SQ0ND6XbxvXB3rrrqKj7zmc902XfNmjVd7tfX13dZe6H7xUbF166rq6O9vb302g8//DBHHXXUHrGEngPDhg3j+eef5/HHH2fOnDk88MAD3HJLMhMDqDM2kzTXTVzF34nK8zHs43bEWWedxdy5c9myZQsA69at44033mC//fbj7bffLu33/ve/n+XLl7Njxw42b97Mr3/d91RbZ511Frfddlvpl8pzzz0HwEc+8hHuu+8+AJYuXcqSJUt6fY0333yTfD7Pueeey9e+9jWeffZZDjjgAIYNG8Zvf/tbAO69995S676z7j9Df6lFn0GlGr1GOgcrHqksdMZWizPPPJMVK1bwt3/7twA0NTXxgx/8gMMPP5yTTz6ZD3zgA0yePJmbbrqJ888/n3HjxjFmzJhSqWdvrr76ar74xS8ybtw43J3Ro0fz85//nMsuu4yZM2cybtw4xo8fz8SJE3t9jXXr1jFz5szSt4lvfOMbANxzzz2lztjDDjuM73//+3s8d/jw4Xv8DP1hafv63tLS4pUalyo9W7LgYcYtuJgXz36IoyeekXQ4VaF9d54jvvooXz7jSL5w+pikw6m4FStWcMwxx1Ts9dZufIctO9o55qDK1Oilq57eLzN7xt1betpfpZsMMnXGxlY8UrpgKowOU7qodJNFVhjbrNJNuLw6Y+PJ2GH60Ic+xI4dO7psu/feeznuuOMSiqgrJfoMKjXo8xpHH0pffuLL0iFbtGhR0iHslUo3WaQrY8umBn0MOr1SQ4k+gywq3aiZGq54qHLK9EF0lNJFiT6LSi16jaMPVarRJxyHSDmU6DNIo27iK42jV6YPZMSp3ZQ74+TZZ59dmkOmv5qamiryOmmkRJ9BlitOapZwIFWkNAWC2vRhLF6JvrdEv3v33r91zp8/n6FDh8YMLns06iaDSslKpZtgatHHE/cwdZ5aeMCAATQ1NXHQQQexePFili9fzjnnnMPatWvZvn07V1xxBbNmzQJg9OjRtLa2smXLFiZPnsyHP/xhfv/733PwwQfzyCOPMGjQIF5++WU+97nPsX79egYPHsxdd93F0UcfzZ/+9Cc++clP0t7ezqRJkyp/EFJEiT6LNOomtuKMzpkYR//obPjLC/16ieHtuzkg7zAwSjF/cxxM7n3FqM5TCy9YsIApU6awdOlSmpubAZg7dy4HHngg27Zt48QTT+Tcc89l+PDhXV5j5cqV/OhHP+Kuu+7i/PPP5+GHH+ZTn/oUs2bN4o477mDMmDEsWrSIz372szz55JNcccUVXHbZZVx44YXMmTOnXz9v2inRZ1GuuMKUEn2o4sVlGUjzqTBx4sRSkofCEoM//elPAVi7di0rV67cI9E3Nzczfvx4oDDl8Jo1a9iyZQu///3vu0wFXLyw6Xe/+x0PP/wwAJ/+9Kf5yle+sk9/piQp0WdQsXSjhUfCZWr2yr20vENt2LSNjVt38oGDDyjr+UOGDCndXrBgAb/61a9YuHAhgwcP5tRTT91jqmHoOpVxXV0d27ZtI5/PM3To0F4X8MjENzTUGZtJlovG0aNEH6pjrptsJIb+inuU9jYt7+bNmxk2bBiDBw/mxRdf5A9/+EPw6+6///40Nzfz4IMPAoVy5fPPPw/AySefzP333w9Qmnq4VinRZ1Bpzdi8SjehOhYeSTiQahHzOHWelvef/umfujw2adIk2tvbGTduHFdffTUnnXRSrNe+7777uPvuu/ngBz/IscceyyOPPALAt7/9bebMmcOJJ57I5s2b4wVcZVS6ySKNo4+t+DtReX7f+eEPf9jj9oaGBh599NEeHyuuJjVixIjS2qwAV155Zel2c3Mzjz322B7PbW5uZuHChaX7s2fPLifsqqAWfQZpcfD4SjN9qkkfTM2I9FCizyDT8Mr4SnPdJBuGSDmU6LMoatGbOmODZWEpwUr+4jdQk34fKed9UqLPoFxOLfq48jXeGdvY2MiGDRsqeE7U6IFKmLuzYcMGGhsbYz1PnbFZVKzRa9RNMK/xzthRo0bR1tbG+vXrK/J6f922i7e3t1P/9qCKvJ50aGxsZNSoUbGeE5TozWwS8G2gDvieu9/Q7fEG4D+AE4ANwMfdfU2nxw8FlgPXuvvNsSKUiuu4SESlm1C1PtfNgAEDulyJ2l///sRL3PrkWtbcMKVirynl67N0Y4VVKuYAk4GxwCfMbGy33S4B3nL3I4BbgBu7PX4L0PP4KHnXFUfdaHhlONeasbGowz9dQmr0E4FV7r7a3XcC9wPTuu0zDbgnuv0QcLpF77SZnQOsBpZVJmTpNw2vjK3WSzeVpks10iUk0R8MrO10vy3a1uM+7t4ObAaGm9kQ4CvAdXv7D8xslpm1mllrpWqE0rtSo1SfwmAdc90o1YcozaeUcBxSEJLoezqzu79/ve1zHXCLu2/Z23/g7ne6e4u7t4wcOTIgJOmP0lw3SvTBNHtlPLlSi17nWBqEdMa2AYd0uj8KeLWXfdrMrB44ANgIfAiYbmbfBIYCeTPb7u7f6XfkUjbNXhlfaXFwDUgOUvzio4Fd6RCS6J8GxphZM7AOuAD4ZLd95gEzgIXAdOBJL/wq/9/FHczsWmCLknzy1KKPL6+lBGMpdcaqeJMKfSZ6d283s8uBxykMr5zr7svM7Hqg1d3nAXcD95rZKgot+Qv2ZdDSPx0jItSiD1XrwysrTZ2x6RI0jt7d5wPzu227ptPt7cB53Z/Xbf9ry4hP9oWO3thEw6gmSljxdJQHEw5EAE2BkEm5qNBs+hTGUDhWWngkTKkzVo2JVFCizyKVbmIrzUevPB9EnbHpokSfQaYCamwdF0wp04foKN3oHEsDJfoMykWjbvQhDFcaR688H6TUlkg2DIko0WdQqVWq0k2w0jh6JfogHesSJxyIAEr02ZTTqJu48qVvP8r0ITrOMJ1jaaBEn0GavTI+V2dsLDl1A6WKEn0GdXTG6nt1XMrzYYrnWF6ZPhWU6DPIStMU60MYqqNGr1QfQp2x6aJEn0G50sxc+hiG0qibeDqm2Ug4EAGU6LOpVKNX6SaULpiKp2PJA2X6NFCizyDLqTM2LtfslbHkSrNXShoo0WeQOmPjKyUs5fkgHVMgKNWngRJ9Bplmr4xNnbHxaLXKdFGiz6CcSjexdZRuJIRKN+miRJ9BGl4ZnxYeialYutH0lamgRJ9BqtHHp9kr49FRShcl+gwyjaOPrfjtR5OahclpHH2qKNFnkGkcfWx5zWkWi0bdpIsSfQZp4ZH4SlfGKtMH0RQI6aJEn0HFhUckBl0ZG0tH6UapPg2U6DNInbHxFdOVxtHHo0E36aBEn0Eq3cRXrDUrz4fRRXnpokSfQRp1E1/H8EoJoYVH0kWJPqPybirdxKALpuIpdlqrdJMOSvQZlcfU3IqhNAWCMn2QUote3xpTQYk+oxy16ONQ6Sae0jh6nWKpoESfUd7pb+lbxwpTSvVhipOa6RxLAyX6jHJyyvMxqEUfjzpj00WJPqMcMN+ddBhVQ/PRx6M1Y9NFiT6jXG3TWDTqJp6OUfTK9GkQlOjNbJKZvWRmq8xsdg+PN5jZj6PHF5nZ6Gj7GWb2jJm9EP17WmXDl3KpMzYeTc4Vj9a2SZc+E72Z1QFzgMnAWOATZja2226XAG+5+xHALcCN0fY3gb939+OAGcC9lQpc+qfQotenMJRrrptYOsbR6xxLg5AW/URglbuvdvedwP3AtG77TAPuiW4/BJxuZubuz7n7q9H2ZUCjmTVUInDpH9c4+piK89Er0wfR7JWpEpLoDwbWdrrfFm3rcR93bwc2A8O77XMu8Jy77+j+H5jZLDNrNbPW9evXh8Yu/ZA3tejjUIs+Hi08ki4hib6nU7v727fXfczsWArlnM/09B+4+53u3uLuLSNHjgwISfrPMNXog+VLwyuV6UOUOmOV6VMhJNG3AYd0uj8KeLW3fcysHjgA2BjdHwX8FLjQ3V/ub8BSGQ5qbsXQccFUwoFUCS08ki4hif5pYIyZNZvZQOACYF63feZR6GwFmA486e5uZkOBXwBXufvvKhW09F+eHPoYhtMFU/EUSzd5zWqWCn0m+qjmfjnwOLACeMDdl5nZ9WY2NdrtbmC4ma0CvgwUh2BeDhwBXG1mi6M/76n4TyFl0PDKODrG0SvVh9Bs9OlSH7KTu88H5nfbdk2n29uB83p43teBr/czRtkH8mqbxuJaeCQWXRmbLroyNqN0wVQ8Kt3E07GImTJ9GijRZ5jpi3UwzV4Zj0o36aJEn1F5cvpeHUPHpGbJxlEtcjmVbtJEiT7LVLoJpnH08RSPkqZASAcl+ozS7JXxqDM2Ho2jTxcl+oxycqrRx6AjFU/HqBsduTRQos+owpWxKt0EK9boVaQP0jEFQqJhSESJPqNck5rFUqw1K82HKbXodY6lghJ9RjmGqbkVTCtMxaM1Y9NFiT6jtPBIPK5RN7F0LDyScCACKNFnmBYeicNLC48kHEiV0JWx6aJEn1GOYagzNlSpZapEH6SY6NWiTwcl+oxyU4s+llJnrDJ9CNMkCKkSNHul1B4nx5gtf2TV1yYkHUoXb46YyEmX3ZF0GHtQZ2w8uagJ+fVfrOC2J1clG0wnOTP+8cwjOfWobM2WrkSfUX856lO89ecFSYfRxXu2vcyY1+f3vWMCOua6UaYPMXr4EM6dMIrN23YmHUoX//U/61nw0nolesmGD318Nh3rw6TDH+74LB987cGkw+iRaxx9LI0D6vi38z+YdBh7+F/f+DVbd7QnHca7TjV6SQ1v2I9BtpP2XelqBUKnSc2U6ataU2M9W5ToRZJjDU0AbH17c8KR7Klj0I0yfTUb0qBEL5Kousb9AXjn7Y0JR7In1xJTNaFJiV4kWXWDCol++5ZNCUfSO10wVd2aGurZsl2JXiQx9VGi37H1rwlHsqfSpGYq0le1poZ6dcaKJGngkAMA2Lk1hTV6VW5qQlNjPW8r0Yskp7FpKAC7tr2dcCR70gVTtaHYos/aHDxK9JIajVGLfve29LbodcFUdWtqqCfvsG3X7qRDeVcp0UtqDNpvGAD5Helr0WuR69owpKFwjWjWOmSV6CU1hjQVOmN9e/oSfZEa9NVtv8ZCos9anV6JXlKjfsBA3vEGbOeWpEPZg2v2ypowZGAh0Wdt5I0SvaTKOzYI25m+Fn1HjT7ZOKR/mhpVuhFJ3DYbTP2u9LXoO+a6UaavZk3FGr1a9CLJ2Z4bTH371qTD2ENxKUGl+eqmRC+SAjvrBjMgjYles1fWhFLpJmOJPmg+ejObBHwbqAO+5+43dHu8AfgP4ARgA/Bxd18TPXYVcAmwG/iCuz9eseil5uysb+KYrU+z5voPJB1KF1PzzpSBjtmUpEORfii26G/99UruXfjnhKPZ06lHjeSrU8ZW/HX7TPRmVgfMAc4A2oCnzWyeuy/vtNslwFvufoSZXQDcCHzczMYCFwDHAu8DfmVmR/ZqUgUAAAWPSURBVLp7tq5WkGC5lpksfzad6+E0NQxIOgTpp8YBdVz+d0ew+s309QMBvHf/xn3yuiGfqInAKndfDWBm9wPTgM6JfhpwbXT7IeA7Vui1mgbc7+47gD+Z2aro9RZWJnypNR887Xw47fykw5AaduVZRyUdwrsupEZ/MLC20/22aFuP+7h7O7AZGB74XMxslpm1mlnr+vXrw6MXEZE+hST6nrqful8P3ts+Ic/F3e909xZ3bxk5cmRASCIiEiok0bcBh3S6Pwp4tbd9zKweOADYGPhcERHZh0IS/dPAGDNrNrOBFDpX53XbZx4wI7o9HXjSC9eMzwMuMLMGM2sGxgB/rEzoIiISos/OWHdvN7PLgccpDK+c6+7LzOx6oNXd5wF3A/dGna0bKfwyINrvAQodt+3A5zTiRkTk3WVpm4C/paXFW1tbkw5DRKSqmNkz7t7S02O6MlZEpMYp0YuI1LjUlW7MbD3Qn2uTRwBvViicd1s1xw6KP0nVHDso/kp4v7v3OD49dYm+v8ystbc6VdpVc+yg+JNUzbGD4t/XVLoREalxSvQiIjWuFhP9nUkH0A/VHDso/iRVc+yg+PepmqvRi4hIV7XYohcRkU6U6EVEalzNJHozm2RmL5nZKjObnXQ8IcxsjZm9YGaLzaw12nagmf3SzFZG/w5LOs4iM5trZm+Y2dJO23qM1wpujd6PJWY2IbnIe439WjNbFx3/xWZ2dqfHropif8nMzkom6g5mdoiZ/cbMVpjZMjO7Itqe+uO/l9ir4vibWaOZ/dHMno/ivy7a3mxmi6Jj/+No0keiSRx/HMW/yMxGJxk/AO5e9X8oTLb2MnAYMBB4HhibdFwBca8BRnTb9k1gdnR7NnBj0nF2iu0jwARgaV/xAmcDj1JYk+AkYFEKY78WuLKHfcdG51AD0BydW3UJx38QMCG6vR/wP1GcqT/+e4m9Ko5/dAybotsDgEXRMX0AuCDafgdwWXT7s8Ad0e0LgB8nee64e8206EvLHbr7TqC43GE1mgbcE92+BzgnwVi6cPenKMxO2llv8U4D/sML/gAMNbOD3p1I99RL7L0pLYHp7n8CiktgJsbdX3P3Z6PbbwMrKKzWlvrjv5fYe5Oq4x8dw+IiswOiPw6cRmHpVNjz2Bffk4eA06OlVRNTK4k+aMnCFHLgCTN7xsxmRdve6+6vQeEDArwnsejC9BZvtbwnl0eljbmdymSpjj0qBRxPoWVZVce/W+xQJcffzOrMbDHwBvBLCt8yNnlh6VToGmNvS6smplYSfdCShSl0srtPACYDnzOzjyQdUAVVw3vy/4DDgfHAa8C/RdtTG7uZNQEPA19097/ubdcetiX6M/QQe9Ucf3ff7e7jKaySNxE4pqfdon9TF3+tJPqqXLLQ3V+N/n0D+CmFE+j14lfs6N83koswSG/xpv49cffXow9wHriLjvJAKmM3swEUEuV97v6TaHNVHP+eYq+24w/g7puABRRq9EOtsHQqdI2xt6VVE1MriT5kucNUMbMhZrZf8TZwJrCUrssyzgAeSSbCYL3FOw+4MBr9cRKwuVhiSItuNeuPUTj+kMIlMKMa793ACnf/904Ppf749xZ7tRx/MxtpZkOj24OAj1LoZ/gNhaVTYc9j39PSqslJuje4Un8ojDL4Hwq1s68mHU9AvIdRGFnwPLCsGDOFWt6vgZXRvwcmHWunmH9E4Sv2Lgqtlkt6i5fC19c50fvxAtCSwtjvjWJbQuHDeVCn/b8axf4SMDkFx/7DFL7+LwEWR3/Orobjv5fYq+L4A+OA56I4lwLXRNsPo/ALaBXwINAQbW+M7q+KHj8s6fNHUyCIiNS4WindiIhIL5ToRURqnBK9iEiNU6IXEalxSvQiIjVOiV5EpMYp0YuI1Lj/D3byEOWwrtGcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.title(\"LR\")\n",
    "plt.plot(trained_logs[\"lr\"] + finetuned_logs_hard[\"lr\"], label=\"finetuned_soft\")\n",
    "plt.plot(trained_logs[\"lr\"], label=\"trained\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VGG(\n",
       "  (features): Sequential(\n",
       "    (0): Conv2d(3, 30, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (1): BatchNorm2d(30, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU(inplace=True)\n",
       "    (3): Conv2d(30, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (4): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (5): ReLU(inplace=True)\n",
       "    (6): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (7): Conv2d(64, 118, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (8): BatchNorm2d(118, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (9): ReLU(inplace=True)\n",
       "    (10): Conv2d(118, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (11): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (12): ReLU(inplace=True)\n",
       "    (13): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (14): Conv2d(128, 253, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (15): BatchNorm2d(253, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (16): ReLU(inplace=True)\n",
       "    (17): Conv2d(253, 255, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (18): BatchNorm2d(255, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (19): ReLU(inplace=True)\n",
       "    (20): Conv2d(255, 255, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (21): BatchNorm2d(255, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (22): ReLU(inplace=True)\n",
       "    (23): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (24): Conv2d(255, 417, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (25): BatchNorm2d(417, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (26): ReLU(inplace=True)\n",
       "    (27): Conv2d(417, 181, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (28): BatchNorm2d(181, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (29): ReLU(inplace=True)\n",
       "    (30): Conv2d(181, 5, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (31): BatchNorm2d(5, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (32): ReLU(inplace=True)\n",
       "    (33): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
       "    (34): Conv2d(5, 5, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (35): BatchNorm2d(5, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (36): ReLU(inplace=True)\n",
       "    (37): Conv2d(5, 5, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (38): BatchNorm2d(5, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (39): ReLU(inplace=True)\n",
       "    (40): Conv2d(5, 411, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "    (41): BatchNorm2d(411, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (42): ReLU(inplace=True)\n",
       "  )\n",
       "  (avgpool): AvgPool2d(kernel_size=2, stride=2, padding=0)\n",
       "  (classifier): Sequential(\n",
       "    (0): Linear(in_features=411, out_features=100, bias=True)\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "finetuned_model_hard"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}