4ks_gridsearch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "from tqdm import tqdm\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn\n",
    "seaborn.set()\n",
    "\n",
    "import reservoir\n",
    "import reckernel\n",
    "import kuramoto\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "use_cuda = torch.cuda.is_available()\n",
    "device = torch.device(\"cuda\" if use_cuda else \"cpu\")\n",
    "today = '200609'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Grid search for Kuramoto-Sivashinsky prediction\n",
    "\n",
    "We perform the grid search on Structured Reservoir Computing as it is the most efficient method for our case. We run it on a time series of length 10,000 for training and 4,000 for testing. The reservoir size is about 8,000.\n",
    "\n",
    "Here are the hyperparameters to tune:\n",
    "- input_scale: scale of the input weights\n",
    "- res_scale: scale of the reservoir weights\n",
    "- bias_scale: scale of the random additive bias\n",
    "- renorm_factor: scale of the current input, that is concatenated with the reservoir state\n",
    "- alpha: L2 regularization factor\n",
    "Since there is a lot of hyperparameters to vary, we perform grid searches on each of them successively.\n",
    "\n",
    "## Data generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "L = 22 / (2 * np.pi)  # length\n",
    "N = 100  # space discretization step\n",
    "dt = 0.25  # time discretization step\n",
    "N_train = 10000\n",
    "N_test = 5000\n",
    "N_init = 1000  # remove the initial points\n",
    "tend = (N_train + N_test) * dt + N_init\n",
    "\n",
    "dns = kuramoto.KS(L=L, N=N, dt=dt, tend=tend)\n",
    "dns.simulate()\n",
    "u = dns.uu[N_init:] / np.sqrt(N)\n",
    "[u_train, u_test, _] = np.split(u, [N_train, N_train+N_test], axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grid search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10it [01:25,  8.55s/it]\n"
     ]
    }
   ],
   "source": [
    "test_len = 4000\n",
    "n_rec_pred = test_len\n",
    "u_train_t = torch.from_numpy(u_train).to(device)\n",
    "u_test_t = torch.from_numpy(u_test).to(device)\n",
    "input_len, input_dim = u_train_t.shape\n",
    "\n",
    "pred_horizon_range = 1\n",
    "out_train = torch.zeros(input_len, input_dim * pred_horizon_range).to(device)\n",
    "for pred_horizon in range(1, pred_horizon_range+1):\n",
    "    out_train[:, (pred_horizon-1)*input_dim:pred_horizon*input_dim] = torch.roll(u_train_t, -pred_horizon, dims=0)\n",
    "\n",
    "n_res = 8192-input_dim\n",
    "input_scale = 0.4\n",
    "res_scale = 0.9\n",
    "bias_scale = 0.4\n",
    "renorm_factor = 1.1 * np.sqrt(input_dim / n_res)\n",
    "alpha = 1e-2\n",
    "leak_rate = 1\n",
    "\n",
    "## Choose here the parameter for the grid search\n",
    "input_scale_vec = np.arange(start=0.1, stop=1.01, step=0.1)\n",
    "# res_scale_vec = np.arange(start=0.1, stop=1.01, step=0.1)\n",
    "# bias_scale_vec = np.arange(start=0.1, stop=1.01, step=0.1)\n",
    "# renorm_factor_vec = np.arange(start=0.1, stop=1.51, step=0.1)\n",
    "# renorm_factor_vec = [renorm_factor * np.sqrt(input_dim / n_res) for renorm_factor in renorm_factor_vec]\n",
    "# alpha_vec = [1e-3, 3e-3, 1e-2, 3e-2, 1e-1]\n",
    "\n",
    "## Change here the parameter of the grid search to vary\n",
    "err = torch.zeros(len(input_scale_vec))\n",
    "for i, input_scale in tqdm(enumerate(input_scale_vec)):\n",
    "    try:\n",
    "        bias = bias_scale * torch.randn(n_res).to(device)\n",
    "        model = reservoir.ESN(input_dim, res_size=n_res, res_scale=res_scale, input_scale=input_scale, \n",
    "                              f='erf', leak_rate=leak_rate, bias=bias, random_projection='structured')\n",
    "        X = model.forward(u_train_t).to(device)\n",
    "        X = torch.cat((X, u_train_t*renorm_factor), dim=1)\n",
    "\n",
    "        output_w = model.train(X, out_train, alpha=alpha)\n",
    "\n",
    "        Xtest = model.forward(u_test_t[:test_len]).to(device)\n",
    "\n",
    "        n_rec = 400\n",
    "        test_rec_pred = model.rec_pred(Xtest, output_w, n_rec, input_dim, concat=u_test_t[:test_len]*renorm_factor, renorm_factor=renorm_factor)\n",
    "\n",
    "        new_pred_horizon = pred_horizon_range * (n_rec+1)\n",
    "        out_test = torch.zeros(test_len, input_dim * new_pred_horizon).to(device)\n",
    "        for pred_horizon in range(1, new_pred_horizon+1):\n",
    "            out_test[:, (pred_horizon-1)*input_dim:pred_horizon*input_dim] = torch.roll(u_test_t, -pred_horizon, dims=0)[:test_len]\n",
    "\n",
    "        rec_pred = test_rec_pred.reshape(n_rec_pred, (n_rec+1)*pred_horizon_range, input_dim).to(device)\n",
    "        truth = out_test[:n_rec_pred, :].reshape(n_rec_pred, (n_rec+1)*pred_horizon_range, input_dim).to(device)\n",
    "        diff = rec_pred - truth\n",
    "        err[i] = torch.sqrt(torch.mean(diff**2))\n",
    "\n",
    "        del X, Xtest, model, bias, output_w, test_rec_pred, out_test, rec_pred, truth, diff\n",
    "        torch.cuda.empty_cache()\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([0.7406, 0.1502, 0.1447, 0.1448, 0.1463, 0.1520, 0.1535, 0.1536, 0.1557,\n",
      "        0.1572])\n",
      "tensor(2)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x7f74e00f4898>]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAD7CAYAAABgzo9kAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAZ9klEQVR4nO3df4wc533f8ffs7P3cPZLH3TmK3JOtWjG/VhPJAiNFcWsjKGI5gOA2LlCkZgvTSAq4SgsBdVugaQvLagMHQWogQBMaUosGoOuUTZAatluotYsibuKgreNWsmwn+kr+IUt3pMW7I8Ujj/dzd/vHzt3tHvd4u8e9m92ZzwsQdve5mb0vH1KfffaZmWeCer2OiIikTy7pAkRE5GAo4EVEUkoBLyKSUgp4EZGUUsCLiKRUPukCYiPAo8BloJpwLSIigyIETgJ/Cqzu/GG/BPyjwB8nXYSIyIB6H/C1nY39EvCXAa5dW6JW6/68/FKpyMLCzZ4XNajUH63UH9vUF60GvT9yuYDJyQLEGbpTvwR8FaBWq+8r4Df3lW3qj1bqj23qi1Yp6Y+2U9s6yCoiklIKeBGRlFLAi4iklAJeRCSlFPAiIimlgBcRSamBD/iXvrfAU5/+Q9Y3akmXIiLSVwY+4FfWNnjt8iKXF5aSLkVEpK8MfMBXygUAZucV8CIizQY+4E8cHycfBszOKeBFRJoNfMDnwxzTUxPMzg3uehIiIgdh4AMe4G33TGiKRkRkh1QE/NvvOcL89RWWVzeSLkVEpG+kJOAnALikUbyIyJZ0BPzJI4DOpBERaZaKgJ+aHGdkKGRGB1pFRLakIuBzuYBT5YJOlRQRadLRHZ3M7DRwASgBC8A5d391xzafBR5qanoI+JC7f6lHtd5RJSrw0nfnD+NXiYgMhE5H8M8C5939NHAeeG7nBu5+zt0fdveHgY8C14Av96zSPUyXCyzeWmdxae2wfqWISF/bM+DNbAo4A1yMmy4CZ8wsusNufwf4XXdfvfsSO1OZKgI60CoisqmTKZp7gVl3rwK4e9XMLsXtczs3NrNh4G8B7++2mFKp2O0uW95tJwC4vrxBFE3s+33SQn3QSv2xTX3RKs390dEcfJc+BLzu7i92u+PCws193eE8iiZYX1mjODbEyz9Y4KffdacvF+kXRRPMzd1Iuoy+of7Ypr5oNej9kcsFdxwYdzIH/wZQMbMQIH48Fbe380vA73RZ510LgoBKucDsvE6VFBGBDgLe3a8ALwJn46azwAvu3m56Zhp4H/C7vSyyU5Wocapkvd79twARkbTp9CyaJ4GnzOwV4Kn4NWb2vJk90rTdR4H/7O7XeltmZ6ajIitrVa4uHtqxXRGRvtXRHLy7vww81qb9iR2vP9WjuvalEm3e/OMmpaOjSZYiIpK4VFzJumnz7k4zuqJVRCRdAT8+OsTkxIhu/iEiQsoCHrYPtIqIZF3qAn46KnJp4RbVWi3pUkREEpW6gK+UC2xUa1y5tpx0KSIiiUpdwE9H8Zo0mqYRkYxLXcCfLI0TgG7+ISKZl7qAHx4KmZoc06qSIpJ5qQt4aEzTaIpGRLIulQFfiQq8ee0W6xvVpEsREUlMSgO+SL0Ol+ZvJV2KiEhi0hnw5e01aUREsiqVAT81OUY+DDQPLyKZlsqAz4c5TpYKOpNGRDItlQEPjQOtOhdeRLIsvQFfLnB1cZVbKxtJlyIikoj0Bny8ZMElTdOISEalNuCnN2/+oTNpRCSjUhvwpaOjjAyHOpNGRDIrtQEfBAHT5YLu7iQimZXagIfNM2mWqNfrSZciInLo0h3w5SI3l9dZvLWedCkiIocu3QEfxUsWaJpGRDIo1QGvuzuJSJblO9nIzE4DF4ASsACcc/dX22z3C8AngACoA+939zd7V253jhSGmRgf0hWtIpJJnY7gnwXOu/tp4Dzw3M4NzOwR4BngcXf/CeC9wPUe1blvlbLWpBGRbNoz4M1sCjgDXIybLgJnzCzasenHgU+7+48A3P26u6/0stj9qERFZueXqOlMGhHJmE6maO4FZt29CuDuVTO7FLfPNW33F4EfmNkfAUXg88Cn3L3jZC2Vih0XvlMUTbRtf9dfKPE//u8M5PNEx8f3/f6DZrf+yCr1xzb1Ras090dHc/AdCoGHgMeBYeC/Aa8Dn+30DRYWblKrdT/SjqIJ5uZutP3Z0bHGH/Elf5OHf6zc9XsPojv1RxapP7apL1oNen/kcsEdB8adzMG/AVTMLASIH0/F7c1eB/7A3Vfd/QbwReCn9lV1D23d3UkHWkUkY/YMeHe/ArwInI2bzgIvuPvcjk3/A/ABMwvMbAj4WeCbvSx2P8ZG8pSOjOhUSRHJnE7PonkSeMrMXgGeil9jZs/HZ88A/EfgCvBnND4QvgP8u96Wuz+VqMiMAl5EMqajOXh3fxl4rE37E03Pa8A/jP/rK5VygT977Sob1Rr5MNXXdomIbMlE2k1HRTaqda5cW066FBGRQ5OJgN9ck0ZXtIpIlmQi4E+WxgkCrUkjItmSiYAfyoecmBzXkgUikimZCHiA6Uh3dxKRbMlMwFeiIleuLbO6Xk26FBGRQ5GdgC8XqAOXFzRNIyLZkJ2A37q7kwJeRLIhMwE/NTlGPswp4EUkMzIT8GEux6nyODPzOtAqItmQmYAHqJSLGsGLSGZkKuCnowLXbqyytLKedCkiIgcuUwGvA60ikiXZCvhy484nuqJVRLIgUwF//MgIYyOhrmgVkUzIVMAHQUClrJt/iEg2ZCrgoTEPPzt3k3q9+5t7i4gMkuwFfLnA0soG15fWki5FRORAZS/go/hAq6ZpRCTlMhjwm6dK6kCriKRb5gL+yPgwRwrDOtAqIqmXuYCHxjz8rNakEZGUy2bARwVm55eo6UwaEUmxTAb8dFRkbb3G/PWVpEsRETkw+U42MrPTwAWgBCwA59z91R3bPAP8PeBS3PQn7v73e1dq7zQfaJ06NpZwNSIiB6PTEfyzwHl3Pw2cB57bZbvPuvvD8X99Ge4Ap0qNgNeBVhFJsz0D3symgDPAxbjpInDGzKKDLOwgjY3kKR8d1amSIpJqnYzg7wVm3b0KED9eitt3+rCZvWRmXzGz9/Swzp5rnEmjEbyIpFdHc/Adehb4lLuvm9njwBfN7AF3X+j0DUql4r5/eRRNdLX9O99+nG9/9bscmywwlE/fseZu+yPt1B/b1Bet0twfnQT8G0DFzEJ3r5pZCJyK27e4+4+anv93M3sD+Angf3ZazMLCTWq17k9djKIJ5uZudLXP8cIQ1Vqd77zy5tbyBWmxn/5IM/XHNvVFq0Hvj1wuuOPAeM+hq7tfAV4EzsZNZ4EX3H2ueTszqzQ9fxi4D/DuSz4cm6GuA60ikladTtE8CVwws6eBa8A5ADN7Hnja3b8B/JqZ/SRQBdaAjzSP6vvNPcfHyQVBfEXriaTLERHpuY4C3t1fBh5r0/5E0/OP9rCuAzeUz3Hi+JhWlRSR1Erf0cUuVKKiAl5EUivTAT8dFZh7a5nVtWrSpYiI9FymA75SLlIHLi1oFC8i6ZPpgJ+ONpcs0BWtIpI+mQ746NgYQ/mc5uFFJJUyHfC5XMCpkpYsEJF0ynTAQ2OaRlM0IpJGmQ/4SlTk+s01bi6vJ12KiEhPKeCbbv4hIpImCvhyHPCahxeRlMl8wE9OjDA2kteZNCKSOpkP+CAIdKBVRFIp8wEP22vS1Ovdr0UvItKvFPA05uFvrW7w1s21pEsREekZBTzbSxboTBoRSRMFPLq7k4ikkwIeKI4NcbQ4rBG8iKSKAj42XS4wo3PhRSRFFPCxSlTk8vwStZrOpBGRdFDAxyrlAmsbNeauLyddiohITyjgY5sHWnVFq4ikhQI+trkmja5oFZG0UMDHRoZDomOjGsGLSGoo4JtUykWtKikiqZHvZCMzOw1cAErAAnDO3V/dZVsDXgA+4+7/uFeFHoZKVOBb319gfaPGUF6ffSIy2DpNsWeB8+5+GjgPPNduIzML4599oTflHa5KVKBaq/Ojq7eSLkVE5K7tGfBmNgWcAS7GTReBM2YWtdn8V4D/ArzSswoP0fTWmTQ60Coig6+TEfy9wKy7VwHix0tx+xYzezfwc8Bv9rrIw3LP8XHCXKB5eBFJhY7m4PdiZkPAvwF+0d2rjWn47pVKxX3XEEUT+963WWWqyNz11Z69X1IGvf5eU39sU1+0SnN/dBLwbwAVMwvj8A6BU3H7ppPA/cDzcbgfAwIzO+LuH+u0mIWFm/taKiCKJpibu9H1fu3cMznG92ff6tn7JaGX/ZEG6o9t6otWg94fuVxwx4HxngHv7lfM7EXgLPC5+PEFd59r2uZ1oLz52syeAYqDdhYNNC54+vqfX2F5dYOxkZ58wRERSUSnZ9E8CTxlZq8AT8WvMbPnzeyRgyouCZsHWi8taB5eRAZbR0NUd38ZeKxN+xO7bP/M3ZWVnMrW3Z2WuP/U0YSrERHZP13Ns0P52BjD+ZyWLBCRgaeA3yEXBJwqF5id17nwIjLYFPBtVKKC7s8qIgNPAd/GdFRkcWmNxVtrSZciIrJvCvg2Ng+0XtIoXkQGmAK+jUo5XpNGSxaIyABTwLdxrDhMYTSvRcdEZKAp4NsIgoBKWQdaRWSwKeB3UZkqMjt/k3q9+7VxRET6gQJ+F9PlAsurVa7dWE26FBGRfVHA76ISr0mjaRoRGVQK+F2cKsdr0uiKVhEZUAr4XRTHhjhWHGbmikbwIjKYFPB3MB0VNYIXkYGlgL+DSlTg0vytfd1lSkQkaQr4O6iUi2xUa1x5aznpUkREuqaAv4PNNWlmrmiaRkQGjwL+Dk6VCwRoTRoRGUwK+DsYGQqJJse0Jo2IDCQF/B4q5YJG8CIykBTwe6hERd68usz6RjXpUkREuqKA38N0VKBWr3N54VbSpYiIdEUBv4fK5pIFWpNGRAaMAn4PJ46PE+YCZnRFq4gMmHwnG5nZaeACUAIWgHPu/uqObX4R+DhQA0Lg37r7v+5tuYcvH+Y4WRrXCF5EBk6nI/hngfPufho4DzzXZpv/BLzb3R8G/hLwj8zsod6UmaxKVFTAi8jA2TPgzWwKOANcjJsuAmfMLGrezt0X3X1z0ZZxYAhIxSIulXKBhcUVllc3ki5FRKRjnYzg7wVm3b0KED9eittbmNlfM7PvAD8E/pW7f6uXxSZlc8kCnQ8vIoOkozn4Trn7l4AvmdnbgC+Y2fPu7p3uXyoV9/27o2hi3/vu5aFcDvgWiysbB/p7emlQ6jws6o9t6otWae6PTgL+DaBiZqG7V80sBE7F7W25++tm9nXgg0DHAb+wcHNfS/NG0QRzcze63q9TQb3OyFDIy99f4Mz9pQP7Pb1y0P0xaNQf29QXrQa9P3K54I4D4z2naNz9CvAicDZuOgu84O5zzduZ2QNNz8vAXwFSMUWTCwJOackCERkwnU7RPAlcMLOngWvAOQAzex542t2/AXzMzD4ArAMB8Nvu/pUDqDkRlajAN787n3QZIiId6yjg3f1l4LE27U80Pf94D+vqO9PlAl976TKLS2scKQwnXY6IyJ50JWuHKlONeS4tHSwig0IB36HpeE2aGc3Di8iAUMB36EhhmOLYkK5oFZGBoYDvUBAEjZt/aIpGRAaEAr4LlajAzPwS9XoqVmAQkZRTwHdhOiqyulZlYXEl6VJERPakgO/C1po0mocXkQGggO/C5t2dZjQPLyIDQAHfhfHRISYnRrRkgYgMBAV8lypRQVM0IjIQFPBdmo6KXF5YolqrJV2KiMgdKeC7VCkX2KjWuXJtOelSRETuSAHfpemosSbNjKZpRKTPKeC7dLI0ThBo0TER6X8K+C4ND4VMTY7rQKuI9D0F/D5MlwtaVVJE+p4Cfh8qUYEr126xtl5NuhQRkV0p4PehEhWp1+Hywq2kSxER2ZUCfh+0ZIGIDAIF/D6cOD5GPgy0ZIGI9DUF/D6EuRwnS1qyQET6mwJ+nypRgdl5TdGISP9SwO9TpVzg6uIqt1bWky5FRKQtBfw+VeIlCzQPLyL9Kt/JRmZ2GrgAlIAF4Jy7v7pjm08AHwaqwDrwz9z9y70tt39MN93d6Z3TxxKuRkTkdp2O4J8Fzrv7aeA88Fybbb4OPOruDwG/BPyemY31psz+UzoyyshwqAOtItK39gx4M5sCzgAX46aLwBkzi5q3c/cvu/vmlT8vAQGNEX8qBUHAdFkHWkWkf3Uygr8XmHX3KkD8eClu38054HvuPnP3JfavSlRgZm6Jer2edCkiIrfpaA6+G2b2M8CvAo93u2+pVNz3742iiX3vu192X4k/+uZlhkaHmTwyeui//06S6I9+pv7Ypr5oleb+6CTg3wAqZha6e9XMQuBU3N7CzN4DfA74eXf3botZWLhJrdb9aDiKJpibu9H1fnfr2Fij+77pb/Lj9x0/9N+/m6T6o1+pP7apL1oNen/kcsEdB8Z7TtG4+xXgReBs3HQWeMHd55q3M7NHgd8D/oa7/799VzxAtk6V1IFWEelDnU7RPAlcMLOngWs05tgxs+eBp939G8BngDHgOTPb3O8j7v6t3pbcP44UhpkYH9KiYyLSlzoKeHd/GXisTfsTTc8f7WFdA6NS1po0ItKfdCXrXapERS7NL1HTmTQi0mcU8HdpOiqwul5l4fpK0qWIiLRQwN8lHWgVkX6lgL9LuruTiPQrBfxdGhvJUzoyolUlRaTvKOB7oBIVmdUIXkT6jAK+BypRgcsLt9io1pIuRURkiwK+B6bLRaq1Om9eW066FBGRLT1fbCyLKls3/7i5ddBVRPpbvV6nWq2xvlGlVoNqrU6tXqfW9NjSFr+u17m9ven55v6t29D6nk0/C8Mc733wJOOjvY9jBXwPnCyNEwQwM7fETz2QdDWSRvV6U7A0Bc/261pTOy2B1PzzmavLXL/e+k2zzo6L9Npcs9fuMr721/bd3thuu1q9TrVaZ6NWo1pt1Fmt1tiIn29Ua422WtxWbXoe77MR71Ntetxo2a75PZvamn5nPwhzAe84eYQfmz7a8/dWwPfAUD7kxOQ4r/1okcWlNcZGQvJhjiAIki5Neqxer7O2XmN5bYPl1Q1W1qosr26wvBo/rm2wsrrB8lp163Fzu9bA3Q7oetvAvn20l0W5ICAMA8JcQD7MEeYar/O5XNyei18HhGGOoXyO0TDc+vnWPk3758Pc1nsemRhleXmNXC4gFwRbj2Gu9XmQgzCXi7dphHLz9rlc6z65ree0ea+gZf/Nmg6CAr5H3naiyNf//Ar/4Le+BjT+UkeHQ8ZG8owO5xkb2Xwebr8ezjM6kmdsOLztcXO/0eGQXE4fFHerXq+zul7dEcTtny+vVlmJH7cCu+nnnWTtUD63/fc4kmdkKGR4KNf4H7opADb/5w93hkSbENhr++Zg2m37yckC16/fuq3egGBnQ5tt2mi73e2NO8c6QcB2SIe5rYBuCeOw8Wc6SIO+XPBeFPA98uGffScPvqPEytrt4bA5yru+tMabV+PR3doGa+udnXUzMhQyGn8gjI2EW8E/NpKPPyS2PzzGhvNMTi6yuHj7Ad/mYGr5Wt7+6Y727Re7BdzO96/HUwVdjVp3zlPuGMXeNsLdMfdZrd8+Al5br3JrZaPtNMNOw0O5rX7d7Osjk+PbH77NH8xbfydNfx/x84Makd2tRqANJ12GHBIFfI8cK47wlx882dU+1VptK/xXVquN5zu++m8/Nn1orFW58dZay0hzUL/Ch21Gm7uORpu/+savh8Ic4dDtX5Gb3+v40THq1VocwDu+Ve34gAxz/RnMIvuhgE9QmMtRGM1RGB26q/ep1+usb9S25nuPHh3j6rXG1/CdX3B3+8bbfLygZZOg7dOWN2pub37eEsZtgvegv35vSvvXcJHdKOBTIAgChodChodCjhaGiaIJxkLN24tknb6PioiklAJeRCSlFPAiIimlgBcRSSkFvIhISingRURSql9OkwyBu7okX5fzt1J/tFJ/bFNftBrk/miqPWz386DeH1dAvhf446SLEBEZUO8DvrazsV8CfgR4FLgMVBOuRURkUITASeBPgdWdP+yXgBcRkR7TQVYRkZRSwIuIpJQCXkQkpRTwIiIppYAXEUkpBbyISEop4EVEUqpflirYNzM7DVwASsACcM7dX022qsNnZiXg3wP3A2vAq8Dfdfe5RAvrA2b2SeAZ4EF3/3bC5STCzEaB3wTeD6wA/8vdP5ZsVckxsw8Cv0rjLpMB8C/c/fPJVtV7aRjBPwucd/fTwHnguYTrSUod+A13N3d/EPge8OsJ15Q4MzsD/DTww6RrSdhv0Aj20/G/j08kXE9izCygMRj6iLs/DHwEuGBmacjDFgP9BzKzKeAMcDFuugicMbMouaqS4e5X3f2rTU3/G3h7QuX0BTMbofGh/8tJ15IkMysC54BPuHsdwN3fTLaqxNWAo/HzY8Bld68lWM+BGOiAB+4FZt29ChA/XorbMyseifwy8KWka0nYvwQ+5+6vJV1Iwu6nMX35STP7hpl91czem3RRSYk/5H4B+KKZ/RD4Ao0PwNQZ9ICX9n4LuAn8dtKFJMXM3gM8Anwm6Vr6QAi8A3jB3R8B/gnweTM7kmxZyTCzPPBPgZ9397cDfxX4/fibTqoMesC/AVTMLASIH0/F7ZlkZp8G3gn8zTR+5ezCzwAPAD8ws9eAaeDLZvaBJItKyOvABvFUprv/H2AeOJ1kUQl6GDjl7n8CED8u0fj3kioDHfDufgV4ETgbN52lMUrJ5JkjZvZrwE8CH3L325YOzRJ3/3V3P+Xu97n7fcAM8HPu/pWESzt07j4P/CHwOGydeTYFfDfJuhI0A0ybmQGY2QPACRonJqTKwC8XbGbvonGa5CRwjcZpkp5sVYfPzH4c+DbwCrAcN//A3f96clX1j3gU/8EMnyb5DuB3aJxOvA78c3f/r8lWlRwz+9vAr9A42ArwSXf/QoIlHYiBD3gREWlvoKdoRERkdwp4EZGUUsCLiKSUAl5EJKUU8CIiKaWAFxFJKQW8iEhKKeBFRFLq/wOwGY+El8FMIAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(err)\n",
    "print(np.argmin(err.cpu()))\n",
    "plt.plot(err);"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}