Approximation/Actor-Critic Policy Gradient.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym, autograd\n",
    "import autograd.numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from IPython import display\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "def softmax(x):\n",
    "    e_x = np.exp(x - np.max(x))\n",
    "    return e_x / e_x.sum(axis=0)\n",
    "\n",
    "def policy(w, state):\n",
    "    return np.argmax(policyfun(w, state))\n",
    "\n",
    "def policyfun(w, state):\n",
    "    \"\"\"\n",
    "    w is SxA matrix\n",
    "    \"\"\"\n",
    "    state = state.reshape(-1, 1)\n",
    "    y = np.matmul(w, state)\n",
    "    return softmax(y)\n",
    "\n",
    "def Q(w, state, action):\n",
    "    \"\"\"\n",
    "    w is SxA matrix\n",
    "    \"\"\"\n",
    "    return np.matmul(w, state)[action]\n",
    "\n",
    "def log_prob(w, state, action):\n",
    "    return np.log(policyfun(w, state)[action])\n",
    "\n",
    "grad_p = autograd.grad(log_prob)\n",
    "grad_v = autograd.grad(Q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARN: gym.spaces.Box autodetected dtype as <class 'numpy.float32'>. Please provide explicit dtype.\u001b[0m\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/Cellar/python3/3.6.1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/ipykernel_launcher.py:21: RuntimeWarning: invalid value encountered in double_scalars\n",
      "/usr/local/Cellar/python3/3.6.1/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/ipykernel_launcher.py:22: RuntimeWarning: invalid value encountered in multiply\n"
     ]
    }
   ],
   "source": [
    "w_p = np.random.randn(4,8) * 0.1\n",
    "w_v = np.zeros((4,8))\n",
    "\n",
    "episode = 1000\n",
    "discount = 0.9\n",
    "alpha = 0.2\n",
    "beta = 0.2\n",
    "\n",
    "env = gym.make('LunarLander-v2')\n",
    "state = env.reset()\n",
    "\n",
    "episode_rewards = []\n",
    "for e in range(episode):\n",
    "    state = env.reset()\n",
    "    I = 1\n",
    "    rewards = []\n",
    "    while True:\n",
    "        action = policy(w_p, state)\n",
    "        next_state, reward, done, _ = env.step(action)\n",
    "        next_action = policy(w_p, next_state)\n",
    "        td_error = reward + discount*Q(w_v, next_state, next_action) - Q(w_v, state, action)\n",
    "        w_p += beta*grad_p(w_p, state, action)*Q(w_v, state, action)\n",
    "        w_v += alpha*td_error*grad_v(w_v, state, action)\n",
    "        state = next_state\n",
    "        rewards.append(reward)\n",
    "        if done:\n",
    "            break\n",
    "    episode_rewards.append(np.sum(rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x112497eb8>]"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYkAAAD8CAYAAACCRVh7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJztnXmYFNXVuN/TPcyw7/sOAiJbQEZwAVREATVxiUbMIl9iJH7RqImJP4zRmEQT/fwSl8SoJPFTYxL3KCpoBHcFFFSURZBNARGQfR9m5v7+6Kqe6uqqXquX6Tnv88wz3bduVZ3qqjrnnnPuIsYYFEVRFMWLUKEFUBRFUYoXNRKKoiiKL2okFEVRFF/USCiKoii+qJFQFEVRfFEjoSiKoviiRkJRFEXxRY2EoiiK4osaCUVRFMWXskILkC3t27c3vXv3LrQYiqIo9YpFixZ9aYzpkKxevTcSvXv3ZuHChYUWQ1EUpV4hIp+mUk/DTYqiKIovaiQURVEUX9RIKIqiKL6okVAURVF8KTojISKTRGSFiKwSkemFlkdRFKUhU1RGQkTCwN3AZGAQcKGIDCqsVIqiKA2XojISwChglTFmjTGmCngEOKvAMimKojRYis1IdAPWO75vsMoUHz7feYDPdx4otBiKUtIcqKph+76qQosBQHVNLY+9u56a2vwsPV1sRiIlRGSaiCwUkYVbt24ttDgF5fhbXub4W14utBiKUnLU1hqWfr4LgPPve5ujf/NSdNvO/VX8/j8r0lLUNbWGRZ9up/Kml/hy76GM5Xpw3qdc8+SH/OudzzI+RjoUm5HYCPRwfO9ulcVgjJlhjKk0xlR26JB0VLmiKEra3PPaas64600+3LCTJRt3x2z7zXPL+ePLq5izfHPKx/vxox/w9Xvm8eXeKt785MuM5bINzM79+fFsis1IvAv0F5E+IlIOTAFmFlKgQ9U1zE3jQQiKPQcPs2Tjrryftz6wYcd+Zi7+vNBiKCXO4vU7AVi/PT6cu+9QNUBankS2z2xtreFXzy5l3Zf7ABCRrI6XKkVlJIwx1cDlwIvAcuAxY8zSQsp06+wVXPzgQhZ9uj3wY9/9yip6T3+eTbviH8If/H0RZ/7xTapragM/bzFw8HANH3+xO3lFD867Zx5X/Ov9vMVklfrJnoOHMSa9Z+RAVU30ubJ18MzFdcGMoJ45Q/rHWbllD//31jpmL/kiEBlSpaiMBIAxZpYxZoAx5ghjzM35PPevn11G7+nPx5St/XIvADv3Hw78fLe9uAKAu+Z+Erdt4ac7rPPv4/qnl3DYw1ik+wIUEz9/6iMm3fFGRsnAL3YfBCKGpr7ywpIvOP53cz3vq5vjfjeXs/70Zh6kqn9UVdd6/oZb9hxk6I3/4b7X1yTcf/mm3Zx+5xtRg3LUDS/wi6eXACBErMSLS+siCfYzZyv5/LTlI9S6LjNPjkTxGYlCcv9bawFiWu+2Gs7lDQmH6g5+wi0v8z8vfEyZVfaTxxbz9/mf8taq+Bjmvqo6JVnfDMZbqyPXU1WduadUn43EL2cu4fNdB9m2N7mR3LTrIIs3NOzQozGGWo9W/JHXz+a021+PK9+4I+KdP//hpoTHve3FFSzbtJt5q7dx8HDkWbQTwl7vfNRIFOB1q3WdVPJkotRIeOBUvvZzmcsbUhaquw0bdx7gz6+uJmw9oXsOHrbkiH8qdx+o824OZaFsC4FtHEJZ/KwHishIVFXX8sKSTTw0b11KCUX7nqfiSWTCF7sOcusLH5dMSG7a3xfR9+ez4sqNiXjbbuzLDiV5wJxb91VVx2wLeViJg9Zza/+qhkgC+RdPf8R+1/6JyMTIuN9x9SQKiPNm2y109w1Zvmk3VdW1HKiKV1R7Dh6Oa+X+7PHFPPXeBrbsORhX3/YanN5AOCyWLJHj3PvaGp77MDbxtedgnZz7PeTIFTv3V0WNl7PsV88uTdkzsOu9vXobn23bH/XePtu2nw+shKEXzjEhBw/X8vLHm9l1IBIq2ObRrfCjDbv4bNv+uPLaWpPWSw2w7PPdvLYytsu1fc/+54WPufTh97jhmaVc8tBC9h5KfOxG1v3NlZH4xdNLuOfV1SxYuw2A11Zu5ZTfv8qh6vwb1kTXeKCqhicWbYj+jks/38U37psX9169tCwS8ln2eSSPdai6JiYRvGnXgZh3zn6+wkkUqf1eG0sWm9pa4xlL+v6DC3lo3jpeXbElep6H5n3Kw/M/44G31/H2qi/pPf15Fq5LnMO05ftk856kdW3cOiVfoa4GbSTWfbmPAb+YzUcbdsUo6Esffo9dBw7z1zfW8JHVw8huyT+2cD0/efQDJt/5BgN+MZujbniB2R9t4spH3gdg8+5ILPSCGfOjx/vP0i94fNEGfvLYYkbdPDdODtsgOFsKtpewZU9E8b2zdjuX//P9mP2civpH/3ovb4N9hv/6JUb+Zg4QeXDPu+dtpsyYbyXV4t37g4dreNKhCACqLMVx1aMfMO62V+h33WyMMYy77RXOvvutaL1tew9xz6urOVxTy75D1Xznbwui21Zu3sP3HljIz5/6iJmLP2fkTXP4yBWW+eqf3mTcba9QU2u497XVbN59kMXrd3LR/e8w6IYX4xoEXiENm9PveoOp97/Dgaoatu45ROVNL9Hn2lms2bqXdQ5D9O66HXz1j7E5hJ8+vphLHqpbHKssHHn1EnmA+6uqeWzh+rjy9dv3M/uj+N/5/jfXRg2i3fDYsS/yjFzzxGJWb92XMLz1l9fXcP69b0e/H66pjbln7pDmxp0HuO3FjxOGOuev2Ub/62bzyoot9J7+PM98sJFH3vmM/7XycTfPWsZPH1/M26sjxuzGmUt5Z+12Fm/wbiicftcbAPz2+eVc8a+69+G4373M5f98L/rd7n20blvkt/py76GYsQn3vraa3tOfj3patbWGddvqPJKfPr7YM1S1fNNubnhmKYdrIvtVVdfSpmkjINLA+eZfI8/neffO4+3V/t1cbYV/6u2vc9698zzrPLZwPcf9bi7LN+3mow27WL89trGTL0+i3q9Mlw2PLVxPVXUtX/3Tm3Rv0yRavnj9Tr7yq//E1LUt/zVPfBh3nP/+R+Th/PnpRzH6t3Ojx9ixr4o2zcqZ9vdFMfUPHq6JKkmoc2udLYVEUYItuw9y/C0vM25A3RiRt1Zt46bnlnH5+H70bNsUEYnmOi64bx7jBnTgspP7+R80TWz5V27eE02yOzHG8OdXV9OhRUX0N+vcqjHHH9EOEYm+ZE6cCtEYg4hw2T/fY/6a7dz6wsdx9T/ZHOlUsHXPIeavibTG7pz7CX+5aCQiEpPH+esba7hl9scsWLONV1bUeQOHqw2jbpsTNcZj+7fn7xePBmDX/sO8tfpL5q3exg1frZtC7KT/fYXNu2MVjru//Nov92GMYdmm3ZxxV53BOHi4hoOHa6JK3L7nW/YcZPz/vsYNZw5i4uDOtGraiP9++L0Yz2Xu8s10btU4erznfjSGXu2a0qJxI/774UXMXvIFv35uGetuOYNWTSKK69UVW9i060C044VIpNXdpVXd825z86zlMd8n3fE6XVo14eHvj6a21tD357M4dVAn/njhCCrKQpxx1xvs3H+YyUO6MKRbq5h9q6prKS8LRccDPGu1+q985INonR+c2JcvdkV+R1up22HdKTPm87eplbRvXsFXerSOOfZtL37MBx45mjnLtzDx9tf52vCu0U4h2/dVRd9PgFlXjKVLq8bcOSfSWcR+FuYs38KT722I1nvq/bjhWZ5c8+SH3HT2EADedOUNv/mXBbw1fTwbdxxgVJ+2iNSFmW58dhkn9Gsfrbtm6172V9VwxSPv88Slx3P1Yx9EZXto3qd5GzjnRYM2Ek43eMOOxFNbHKquTRprth9Mm9PveoNG4XhnbeD1L8R8n7NsM+2alfPPFB6EDTv2M+bWVwB4+eMtMdueen9j9OEe3actj/7gOAAWrN3OgrXbAzUSAH2ufZ6BnVvGlL3xyZe0bNyIZz/8nKfei33Rnly0gW//bQFvXHOy5/E+dLz4ew9Vs+jTHVHl78Xtc1YCEeNjt6rmLN9Mn2tn8fDFo/m2w+uYtybSUnUaCIDDtbVRA2HLD/DCkk1c+nCdchnbv+6FdhoIgMcWbsCL8+6dx8hebWLKLvzLfN7/bCdDLaX6yZa9vPzxFv748iogonSuefJDPrjh1LjQ1sUPxi7Te+Yf32TCUR3569RjYrpFrtqylz2HIkbh8UWxsr29ahtXP76YG786iNdWbmVIt1ZcfdqR/OGlldE6yzftpk/7Zqzeuo/VW/fxysdb6NyqMRAJ+wy8/gXOPbpbTI8/Ywx/fWMtTSvCVJSF+enji7lzynC2W+/M6yvjZ0ZYuXkvuy1veOGnO7hz7ics/byuW7R9vS9cNTZmv7tfWR13LJsVm/fEvYdObE/EjdNApMv6HZEWvpcOOSHBbAinOhLu43//WvSzc2Q34Gsg5i7fwrRxR6QlayY0aCOx+0DqMemq6tpo1zg/nnC9kJt2xecfvPhky15uen558ooQNRDJWLA2+HEdboyJKBQnTyzaEPc72NgGzKvLL8BzDvd+294qT6/Ni7Kw8M8FsS/Sw/Njl+/de9D7Xj80L36Z34fnfxp3r93eYCos+nQHi1xe1vuf7YzKDN6eKcDv/7PSs9zNqi17eXFpbL/5KTMio3q9uPvViDG68dllQMRonjOiW8w9mXznG1w1oX/0+3cfeJeKstjGjrMBsG1fFX2urUsqd2sd8VKcXoOXPF+/py60NSNBV1W711Gxct9ribvZ5ooFa7dTXVMbDV3migadk9h1IPWxD89+uClGiWXCXReOSHuf568Yw/NXjPHd7nbF6wN+LW/n/dhXVc3RPdt41nPj9liAaAvVxi+R7GWwkjUGgsA2Fn48+2Fqo3PXbdvPD1wGzM9AAKzZGt8T6FOPxP48K0dgkyh3MvX+d2K+u3/7bMkkuX9M7zbRkJsfLSrq2shnDuvCaYM68e8fHk/vdk0BGOMIB/3mrMHMvPyEhO/blGN6cO+3j+aMoV1SlvOh741iyjE9klf04eMv9mS8b6o0WE9i98HDvLA09ZGLXu6ykx+edATd2jThun/7K5ivfaUrkwZ35sZnl0ZbvhOO6sic5bFho4mDO0UH8AzuGglLNAp7x/HbNvV/EVZu3sOATi0Syl2sHDxcw/7DNYQkcX7Gj30uo5Cst1ExMbZ/+2jYy0nLxmXs9vGIsmGDxyzCiTzR1392Mp/vOsAUR+eMXJLOeJhxAzowqncbzq/sQaeWjfnqH9+Mdj6xsXMD44/qyKZdBzlzWBcuOq53dPvMH41h5Rd7GNmrDU+9t5Ebn13KN47pQUVZmGcuO4GaWsMRP58V92ze8vVhAEwa0oXnXYNyndxxwXBaNWnEg/PWMaZfe8YN6EDX1k1iQn6p4s4F5YIG60n85NEPkldKATsWvr+qhm+N7hXXenG3EsrLQox1tFDuunAE/3PesOj33507lD9982ievXwML/14XLTcNhA/Gh+bVzihX3suqOzBA989Jk62jTsOxPTWqak1/GPBpzFufrFy8HAt+w9V0755RUb776nHRqKDzzV3a9M0J+e7Pk3PSQTPXJuN1/iCbDiURrhpePdWXD6+P51aRnIoz/5oDI9fehzjB3aM1rlwVE8gEtJ87AfHxRgIgJaNG1HZuy0iwtdHduejGydSURaObg+HhJevPjHawcGLN645mWcuO4HJQzrz1vTx0fJ1t5zB2SO6cfLAjjzw3VHRcRxXnNKfV396Eot/eVq0UwPAKQM7clSXSN7v+2P68MpPT4pu++cl/ucPkgbrSXyyZW8gx+naqgkbdx6Idq+75dyh0d4Ub00fT7fWTdh7qDqmJ8PQ7hHrf0K/djQtL+MblT2isWn7Abbr2Jw5rAvPfbiJn5w6gO37qviH5YmUhYRbzxvm2Q2xutZw2DGWf++h6oSeTjFx8HANew9V06ZpeUxiOVXiPIkctMBzRbMK79eyW+smcTmgQiAC5QmNRLDnO5jG2I5WTcvjyo7p3Zbh32nNDc8s5Vuje9K0PMw/F3xGv47NM5apb4fmlDvyNB1axBr2Hm2b0qNtU+759kgA/nXJsUl/l97tmwEwqGtLPtywi6W/mkiTRmH+s+wLLn34PXq1b0af9s144aqxtG5SHu1MkGsarJHwGgSXCScP7MDD8z+jZ9tIK2+yIx5pv0h/+ubRMft0b9OUO6cMZ3Sfdimf5/YLhnPT2UMQEX591pCokbBjxSLCmH7tY7rhHaquodoRoqpPremDh2vZX1VDj7ZNWJHBJLz7DsXe3+p6NPK4eWM/I5EfpZAMEYkm3r0opCfhR6NwiN+dOzT6/akfHs+gLi0T7JGc7m2a8vb08XRu2TjpyO7jjkj9Xf+//zqGjzbuijYWJg7uzD8vGc2xlr5w9yjMNWoksuCWc4fyjcoenDOie7RLo5PyMv/W1lnD01twr1E4RGurlRQOCW9cczLT/r6Is0fUHcc92nnvweqYPuL1qTV98HANBw7X0LNtU0ae0ta3R5Qf9ckgumnu40l09hjbUAiExOGmoKewTseTSJVUO0Uko2vr4O9Ju+YVnHRkXXhMRDj+iPYJ9sgtDTYnEcS8P+cc3Y1QSBjZq42nQXB3GwySHm2bMvvKsdHYK8CInrE9L95avS0m4V5Mcx0l452126mtNYRDwpCu+W05FZqm5WHPcttbLTQhkbyGm9LxJPI5K2tDocEaiWzDDy/9eFxMMsuLRC9SLvjpxCNjvh9yGYX6NNnbowvXU2MMIZEMZt6v34R9tGyrJo3o0bbw3oQINCrLY7ipnk1eWWo0WCPRpFFiBf+ziUdy5jD//s79U+hamixO6aR983I6t8wu5twoHKJvh2bR71Wu/uXOmWQTzVFULNTWWkYiBVF/c9bg3AvkwclHBr98rl+4pkl5OLqmwJ1Thgd+3lRJFm4K2pN44O21KdfN13xGDYkGayRuv+ArCbdfdnI/z3hj9zZNYrqsejFxcKe05Vnw8wkxXeUyxdl9zp2jcHoSNUW+/sSw7q2oNXarNLmsyby6XDH+qPTvdTL89FzT8jD3fnskZw/vypnDunKSy0Ad27dt4LJ4ISI0Cvmrjs9TnGkgVdzToCj5pcEmrlNJrrmr/P3iUYztn7zlePc3j45rxSfDL8SQLs51L9xGwj1mIokzVRCGdW9F0/Iwew5WU2sM4RAp9QKraFSY9k44B01X5yFbNC6LTgnftDxMr3bNuGNKZOS+uzVfnidDmSzcVEiKU6r6TYP1JFJ5ufu0axbzPdUcQ1k4RNPywthf52W5DZXTe/BaxCgoBnbOfJR3j7aRWU1rTcSQhURo06ycq08dkHC/ts3KmfGdkQnrBJUjck6T0r9T5n3t/XAaemfjoYkroV3maliUJ1s8ISBCIgnDTUpp0WDvdCot9wuO6REzqjFR3/BiJGG4qUhzEkIkpl1bazCmzuNLZtM7tKhImjB1K1k354/snlS+F64aG9NF9ags+9p74bwMZ2PGnUdzT+yWL8Vt5ySmHtcrYb1c5GuGdEv8ewfd/VZpwEbCL6l814UjuM3KObj7J4cTxGGLBedL4p78y+k9bNp1MO2J01Zt2ZtwIRWbbJwUQ8SA1xgTDTdB/MvfyGWw2zWrINntcXYt9Yrf+xmZ4/rWhbvCIjH1Uo0S/vCk1Kd0dh7S+Zy6vdNGIfdvEv8DXDE+2Onhoc6IXZrkmpo3TjzBnhftm1fETLznpmUGx1Syo/i1Xo7wCzeN7BWZHMwLt3tfjCQS0WkTTrv9da596qO0jj3hD6/xzb8sSFrPZNlpNSRCba2JdoGFeE/C6Qn++VtH06FFRdJWpLMl/r0T+sRt99u9sSPf4W5cpNrdM52ck/OY9nP6g3F9447h9mw9B28mkO+35wz13ZYI+3dOdk2ZON6ThnRKmFhI1gBRRyJ4GqyRcLc6R/RsTccWFb6Tq0FwyeVckuglqamN9Ry8lsAMgitO6Z+8UgJsT8KYOoUpLs3hNPKnW1Oh2HX9jHmFw0h4KXc/I+P0IMMiseGgkPCYtbgTRPIe6245g6cvOyHmGGmNHXBUtS/FSza3Z+vlSSQ6q3vwZarYoiS7pkzGS6gRKD4arJFwexITB3fmnesmJJxKoz54EoleMnd0KVfzGZ0+JPX59N1EchISnXPKz5PwChfaRX6hxCYxHkH8dr/b69S94ZDEKOyQCEd6JOqd4bA7LhieVgPDWdNe/9xLObpDbukmrjN9nu29khqJHLwvaiTyT86MhIjcKCIbReQD6+90x7ZrRWSViKwQkYmO8klW2SoRmZ4r2SAzryDXK0AFgTtZ7WR/Vex8RjW1hikz5nHBfbELsc/6aBNPOlaX27L7YMz3ZGTzIhsiysfOl9g/uft2eSmoZJ6EM3Ht1TL3U3rOZyUUkhhZQuL9LDl7UrVpVp6ekfAIN3ntXuaydOk+n5l6xqEEMsXWy+jwCb2fpJ5GZqdUEpBrrXe7MWa49TcLQEQGAVOAwcAk4M8iEhaRMHA3MBkYBFxo1c0JmbRy6oMnkWh8xs9cS2VW1xrmr4msf719XxXDbnyR9z/bwQ//8R5XP76Yl5ZFpl/97gPvcvXji1OWIZUeJt0STIwWDtV5OeIXbvK4F/Zp/ZSfMyfhVcNP7Lhwk2NvEYn1Sq2PrRyLQbmT3cmISVxHFXL8/m5PIt3eTWWhEJeeGEk+927XNOUkty1KsvuciRHyOuScn4xjwlEd4zcoeaEQTeOzgEeMMYeMMWuBVcAo62+VMWaNMaYKeMSqmxMyGQRVH3IShzOc5+Y7f1vA7oPVMev1XvJQZCH6TNZzSIbf6HI73GR7Er7hJo9bkcyTaJwkJ+HrSTg9h1C8LF67dWxRN8VKKBQbskqGxJzPvn6vnERsmd/EgInOM33yQNbdcgav/uxkxqQwUBRS9yQy7Y7qdhb6dWxBd2vBpaSLUGk8KnBybSQuF5EPReR+EbHn5u0GrHfU2WCV+ZXnhEySavXDk4i8YmP7pze18NLPI4vZhD3i2vm8akNEMdo5Cb8usC08ukLa9zQlT8KjSsqehLi3e+9oz9qatifhkbj2DDe5LE8mRiLR92QkT1ynd7xUGNGzNfclGTSpBEtWRkJE5ojIEo+/s4B7gCOA4cAm4PcByGufd5qILBSRhVu3Jl57Oghs5ZKLRFzQVFlz7985ZQTN0lQaEO9hvb5ya+CzeqYiQ3Wty5Nw1bnjgvgJ7uzb42sknDkJD9Pnn5Nwfpa4ff32s7sCh0KScRdY+1ye4aYUxkkkwm14UxUx1d5NXr+xk/evP9Vnv8SM6effACr+N7T+kZWRMMZMMMYM8fh7xhiz2RhTY4ypBf5CJJwEsBFwDkTobpX5lXudd4YxptIYU9mhQ/CjOt1MGRURK5frQwTFGMuDaFoezsjdX701dlnXi+5/J+8efDgk0TW9vcJN3Vo3oWPL+LCDfb3uhK5Nspl//S7T6UmEQvGehJ9ytZOsIUnPSMTIlMCTcHt9XkvYJjx2CiVe+IUA06FP+2a0aRa/1CgQs+BOovMr+SGXvZuc/SDPAezFlWcCU0SkQkT6AP2Bd4B3gf4i0kdEyokkt2fmSr50+MUZg1j4iwmeIY5i4w/fGM7LV59I40bhjFpVdtjJSTavZLoz2wre4RanDMYxyC5m3ySehNPIew34c3qKvz+/bpbgGE9C4tvHfsbY1tnhULrhJq9r8/IkYl/fdDs0Zxpusqsl9SQSbH502rG+2247fxhvXHNyTJnTACY6rtqP4MnlLHT/IyLDiTy764AfABhjlorIY8AyoBq4zBhTAyAilwMvAmHgfmPM0hzKlzLhkCRPmBUJjRuF6dshMulcUC9MNvPhNE7T+zLEhrzCHolbg0/vJOu/X+4o0RiYyDnqPjtHM7vlSbclG07Tk3DWtK/b65zuEdfpTofiNnepSlgnU3rnc9Khhf/7VFEWpofPKnxCfCOgX8fmrNqy17O+kj058ySMMd8xxgw1xgwzxnzNGLPJse1mY8wRxpgjjTGzHeWzjDEDrG0350q2hkJQOZRMbMRb08fzwlVjEypUv+S6UwnYCskdSklkuHxXdmvqHd6w8ZPVPXgu3d9DJL3edF5VU0lcQ2SxLCeJ7Ea8J5FquMn+nywnkejcmT+b7vPO+cmJHH9EO+uc6koETfEH2ZWMSfV1GT8w+Bhwt9ZNGNi5ZUKF+peLKrlkbJ/484VilTLEKrvIdB3+x3UaCed4jDZNE4cLY0NaDnlcnkSqisg2bOGQJDTYo/rETjYYMw7DQwYbt8dkSJ53iT1PLKknrlPLSQQxI6vXZH9ecmqYKXeokShhEin3b43uGf2crOtkolHcyUikUBs3CtPOI4znVAJeHXYM3jmJun3qts26Yiztm0c8iLZJPAn3ObzkCQlImm9NOCQZj5OoG7gWXy/OSHjEmxLqTrcnkWYrPNfTcv/z+6N58cfj8n5eJZYGayQSxURLhUQvUztHzxJ7gjw/vtid+XKUyRSq1+A/Z2imLtxUt71uWVNvYsJNAtv3VQHQtnkaRsLpSbjCX15nnjykc/wx7P0lPW8s5GEkvEdcZ/f6xuUkEoh477ePzupcfkyfPNB32/H92nsuIZwItR/B02CNROdWjZl37XguHhMf7igVEr0w7R1Gskurxqy75YyY7ZMGxyu9jGRIst1rTQunUvaK5RuT2PjE5hAiRgWgV9tmPnvE4zQSqcTuvzU6fgGelLvAxjkAznCTf5I420Ww0lGok5JM2nj28K4ZyXDCEbF5Kc0pFB8N1kgAdGnVpK73TIFlyQWJ9JJTqXkpsOaNg+n4liw0YI8Qj9YnttVs9/KMrZU43OQ+v71Cmj2YbmSvNlEFfmQn76VWnedznyvV2L0dsgqHJPvEtcdJ48NNKZ/Ck2zGH3zNw0ikcrh0T5nsEkvxPS40hVmIWckLfi/9uAEdYmc29aiXymjtOy4YzrVPfcSBwzUJZEh8DLcnYa9Ml0i2ZIlr9/n/OvWY6Hk+/s0kwiFh3uptAHRsWcGKzXvi9uvRpknMMZx4tXYTKbuQJE5cxx3LJX/k+F5Gwj1OwqQVr3fXzCZUk87uIHNoAAAek0lEQVQIdqV+0aA9iVLH7xV16ysvT6IihV4yKbUUk7TtvJLiscuD2jkJ41sn2fnDIYlO7te4UThpLP/bx/ZkgMPDcJ8rVd0XDTf5jK34SvdW3jJ71E0l3GRMvDeRuAtsZtflfbD4olTmOnOf029Vw1CKHr/apeBRT6KE8WtVuhWW18vsnoba7zjJu0Em3j66b1v+Pv/TmDKnDvf0JFI4brLzJ1KeXVo1cfUwiny5wGdZW/BWXvY5IhP8xW77xRlHAbB4w66Ex7KP4XWP4j2J9IjzJAIO1qTiPaV6zqtOGcDBw7VccEzP5JWVQFFPooRxKrpHHNMgxC3g42kkkj8aqbTmk1U5c1hXjuhQl1B25yQ8u8D6TMuRyfl993MoL/vn6WTNF5Xo3DEKPpq4Tpz8dreenXWdyW83WY+4jpMpvf1j9vUosw1bopHuqZ6zVdNG/O7codG80k9PG+Ajh7oSQaNGooT5yal1L1L/js2599uRKZbdHoZXUjXZFBZgKb8kdVJ5aVs7xi/YK9NF9/foAus3LYe3jBlbibhj2CJ4HjKpR5W6HF4ye4UE3d6e29iM7d8+8ajnuNls4+tMHNwp4zXLbZl/MK6vvwyuc7ZMcX60cQNiJ/bMNmmv+KNGooQ59+ju0c/hkNSNAHYbiSTLb/qRiuLLZGYQpzx+XWBTz0lkhsQYicj/Wuv3S/2YJipEWnJ4VPa6R3HhJlMn9/iBHZnxnco0zxN/jmnjjohpbPgeKsGI8JoEa6k7DdV1px+VskHybXyoIxE4Dd5IjOodmRJhWPfWBZYkt4gIR3aOJGMnD40dA2GHm5zLV7ZOYXRyWUiSGopMRsfGTMsR7QJbp2iMMZ6t+QGdWtC+eQXXOOYv8jMmR1m/hXPkuRPnXvY11CYI/XgpLWPqtiX6HY7s3CKaxG7RuMyVk6jrRuvGq8w+Z8+2TWPWz/AiyHCTF/bcUomMhPMSLhnXN2b1wETkWnaljgafuJ4wqBPvX3+q79z2pUJIoG+H5qy8aXJcKMlurTvXTTh7eFfeWvUl/37fc0kPIBITTx5uykxWm4qyeKURaTHHH7lZRRkLfzEh9vw+AnRsGT+AMFYGifsc9STSvCiRxB5VRVmYZy4fw9ov99GycRlLHNO120rfu3NBtiOuE3+HNDoIeJTZMlcn8iRUuRc9Dd6TAEreQEBdq9Mr1+AZygiHko5GTy0klVw2d/dWZ4ipcaOQVadu+1d61Hl9foPh6s6fmRbyCjdFPQMvT8LjNF8fGQn3NSsvSyk306d9M9o1r/CsmUriOv3lS905iWA1djiFcFOm8SE/UdXmBE+D9yQaCqnMdeSukmzaB6+pqt1kG26yPQlbzYzu05Z7rTWOn77sBHr5rDuQLbG9m1JRdvFMnzSQqyb0p0l52FOppZJsTWSY3IsOffvYXjzs6k6cjnr2kjHVu+e1b0o5CdXqRY8aiQZCopfRb8qIZIOhysLpZmSTI8QaNHeMeniP1jS3po8e3iN3eaTYcRKR/7UJtLpn6z8kNC0vizteKvvGHcsrme0y4umvce2WI2hPIiJPwnBThsf2k1VniA0eDTc1EBLNHeSzJLTvWtE2jUKhwN37yLQcdd/tcFMhqRv1nfkxMl0Frm4mWS9Pwj9xnQmenkQaI9vdRHMSHpM4pnv8+P0y2k3JgMK/gUpeSHlq7RTKbbKdhdQPL08i3/3gvXMSicImiX8L90+Z7XrUkNpYlkRS5XrgWSo5iYw9Cc1J5A01Eg2EZBPQeeEOXzx7+Rhu/fpQx/bkXWAzwWmcokbCMeYgH8TkJEKxXWAzOl6Gv5NtmLzukVd30XRO467rNfI+m5yEvWZLl9aN09ovpfOpOcgbaiQaCImUlB0WsBvKP7LGS7g9iaHdW3HBMT2j01Nk2wXTxq17YzwJV2s5X8rBa+6mhDmJJGK5t7dq4j+y2OssXkaiIgVPIh2C/mXH9m/Pfd8ZyZWn+A/GC/p+ahgqeNRIlDi/O3covdol7gHkF1bym+TPDh+UhUM5eSmdCtHuQZX3cFOMPJH/2XgS9jGGdW/F784dyrkjuqUnTxb5glSP6X2OFI/l+j736hMRESYO7hzI3E3J9tNpOXKHGokS58JRPXntZyd7bjuhXzvAX9n4GQ/bSDQK5aZdb5/3RNf8PJC/lmLs6naWJ5FVbF2ix7pwVM+01peIyJNW9SiJu8C6k+nB/bhHdGge8/32C77Cb84eEi9DxuEmn3L1JAJHu8A2YP5yUSWbdx/y3e4MJzlnaq12eBK5IJP5noLG25PIvLlqGz6n4fVbO8FJollgsyUuJ+E5TiJDV8LFOSO6U1Vdy/VPL3HJEEzvJjUOuUM9iQZM0/Iy+rR3TNPtetGcCm3u1SdFP9eFm4J5M0f0aBPzvRhWNPPOSaRW34th3Vpx8Zg+3DlleFr72oYkJ0YiaUGweHmmmZ+y8M9IQyErIyEi54vIUhGpFZFK17ZrRWSViKwQkYmO8klW2SoRme4o7yMiC6zyR0Wk9OfKKHL8BtPZnkR5OBRI76ZrTx/IZScfUVdgdzl11DFpz8CaHV7hpkRdYOv28y4PhYTrzxxE9zaZjRDPhXcVtzJdmsuyJtvXjaenElBOIh05lPTI1pNYApwLvO4sFJFBwBRgMDAJ+LOIhEUkDNwNTAYGARdadQFuBW43xvQDdgAXZymbkiV+BsCesymcQU7iKz1ax60v0Cgc4sjOLevO67Ff3fQUaZ4wAEIeRiue3AqW7nWnYlTd27L5bVPZ13POK1XqRU9WRsIYs9wYs8Jj01nAI8aYQ8aYtcAqYJT1t8oYs8YYUwU8ApwlkadnPPCEtf+DwNnZyKbkjif/+3h+NvFIzy6w/To299ijjmcuO4FrTz8qpfN4tdwLoVTcs8AGRbZzNwHMvnJsxuePn5Yj/2jiuvjJVeK6GzDf8X2DVQaw3lU+GmgH7DTGVHvUVwrIwxePZsf+qpiyIzu3iK5N4ca5TGo6ON9tL4VYyB6OdXM3Ja+T2fFjd3Yax0TTcgAM9LkPqYQBg5gF9pZzh9KsoixjA5PxfmoN8kZSIyEic4DOHpuuM8Y8E7xIyRGRacA0gJ49dWH0oPBq2Y7p3z7hPu53tX3ziqzlsA8Zs2RpAcNNQ7pFFgSaOLhT/k9u4ZeTSKYs0zGumYyTmDIq8v4tWLMtjTM5T5DX3ZQMSGokjDETktXxYCPQw/G9u1WGT/k2oLWIlFnehLO+l0wzgBkAlZWVOoymoATzujpvYrE1Eo/o0JxPbp6ccIR5rkT+7TlD+cNLK3w9Nz/cobqrJvTnb2+sZc+hap89sgvlZdyVNeD1JJTgyVUX2JnAFBGpEJE+QH/gHeBdoL/Vk6mcSHJ7pok80a8A51n7TwUK4qUo6ZHKy/ry1SdmdOxUxhEEwbOXj+HRJGGyoKYgceJ3dU6FO6x7Kx6ZdpznCn2e+2ZwvkQ75zoHFLSy1zBU8GSVkxCRc4A/Ah2A50XkA2PMRGPMUhF5DFgGVAOXGWNqrH0uB14EwsD9xpil1uH+H/CIiNwEvA/8LRvZlOKhfYvkIaiYnAR2l9O6Mttg5EIFDLXWl35t5daMj5GNcnLv6fQCgtR5yRLvybrZ/vacoXy2fb/ntkzlzHT8h/aKyh9ZGQljzL+Bf/tsuxm42aN8FjDLo3wNkd5PSoHIVSMs3cPacjh12hlDu3DHnE84Y1jXwORyk47f8quvDeaoLi2TV8ySdJWo3zUYk9xIJFuW9Zujg8//ZZ64DlQMJQE6LYeSFam8q6m0spMp6P6dWrDuljNSkikfTD2+d8z3XOmsbI/r3D9Zl1uvc2U6wV/CuuL9OQjUdgSPTsuhZMWVE/onrZO2J2H9z1dOophJN4yVMCeRzEjkQcM+9oPjeOOak+vOqYnrokeNhJIV3xrdi09unpywTkqjcT2+1Kfpn4NUWjE9vQI8VjKj6zktR4oSpHr9o/q0jZ2aJONpObx3VOMRPGoklKxJ9l6mElePVYz6ptsEpfREkq+HUQgFq0q9+FEjoWRNrrod1iNHIlDDlmz0eaYkm6Awm0WHMnUJgh6prQ2M4FEjoWRNstcy3XBTV2tN5OOPaJexTPUBP52djXG0uxt3btUk7eMWQsEGtZ6Ekju0d5OSNUnXd05T+fRq14w3/9/JdPVQdMVKVkorQIV3xtAuhL8lnDY4fiadTBLXqYoW9ER9yffTnES+UCOhZE2y1mAqL65bf2W67kKhKXS4Q0SYPLRLXHkqnQAKsdhT0OtJKMGj4SYl5+j7XDjSGr/gVZaDcRJOMh9xHawcij9qJJSck9K01QGfszxH62/ng8pebZJXSpFsZ4HNNV5LmirFRf19k5R6QypqIOieTO9cdwoLfn5KwEf1JzqVSABX0qJxI3428UgaNwru9RSBOy4YnnAqEW9jnuo4icyUfTjfyQwlbTQnoeQcWw+cPtRrWZLc0Lpp9kukT588kGHW5H/JCDoXcdnJ/bjs5H6BHvPsEd04e0Q3ek9/PtDjQhbhpgw9CU1c5w81EkrOERHeue4UWjfxV9zF+G5feuIRhRah4BSr0i1WuUoRNRJKXujYonGhRcgpmSitUpqbKt9K2/90aj2CRnMSSpTOLSOKvHOr/Cv00lGXxUmm82CpylXUk1CinF/ZndZNGzHhqMKt5VxfyaYlncuxFflS8vkeH6IT/OUPNRJKFBHvkbp5OXdBzho8hR5M5yZbD61YlwMtTqlKEw03KUoAFJtxcJNrXZ/3nITP+Yr7LtRP1EgoisV3ju3FUV1acv7I7oUWpWgoVqVb7Ea5lNBwk6JYdG3dhNlXjs1o3yKNypQufp6E3ojAUU9CURRfVOcqaiSUoqC+d4HNRJfWp+VZk1EsOQkleDTcpCgF4qLjerHiiz1cemLfnJ8r83ESxamNbancxqI4pa3fqJFQioL6/nJn0rJt0bgRd104InhhCkChxknU9+emPpBVuElEzheRpSJSKyKVjvLeInJARD6w/u51bBspIh+JyCoRuUusuy0ibUXkJRH5xPof3HzJitLAKbXFfeo8iVgBi1Xe+ky2OYklwLnA6x7bVhtjhlt/lzrK7wEuAfpbf5Os8unAXGNMf2Cu9V1pINT/8HzD1k5O5fzs5WPyd968nanhkpWRMMYsN8asSLW+iHQBWhpj5htjDPAQcLa1+SzgQevzg45yRVHqCR1aVDA0xenVs8E2SnE5CbUagZPL3k19ROR9EXlNROzO592ADY46G6wygE7GmE3W5y8AnUCoAVHf3+2GrpzyPwtsA//B80jSxLWIzAG8JvS5zhjzjM9um4CexphtIjISeFpEBqcqlDHGiIhvBEJEpgHTAHr27JnqYRVFSZNiNX5RT8JlLNR4BE9SI2GMmZDuQY0xh4BD1udFIrIaGABsBJxzHnS3ygA2i0gXY8wmKyy1JcHxZwAzACorK+t/OFup9xSraqooiwQLGuV4ze+CKedi/eFLiJx0gRWRDsB2Y0yNiPQlkqBeY4zZLiK7ReRYYAFwEfBHa7eZwFTgFuu/n5eiFCHdWjfh6zrnUdG1vC8e04d9h6q5eEyfjPZPdZqLQl133GmL7PcvBbIyEiJyDhEl3wF4XkQ+MMZMBMYBvxaRw0AtcKkxZru12w+BB4AmwGzrDyLG4TERuRj4FPhGNrIp+eWt6eMLLUJBKdY5gxo3CnPNpIGFFiNwaq3RgUX6s5cUWRkJY8y/gX97lD8JPOmzz0JgiEf5NuCUbORR6i8aMyxOUtXB+dbV5Vb4bOpxvfN85oaHjrhWlADQBm3mzLpiLDW16TUTysIhVv/2dEI6LUfOUSOhFAX6chcnqYZzsgn7DOraMqP9wm4LoeQEnQVWUQJAY+PFQbHmhuoz6kkoSj1nWPdWjO7TNifHTr1rqyrnUkWNhKIEQCEHcc3M41xJxY6aquDRcJOiKL7kIyehFDdqJBQlAEpVSda3yyrV+1BI1EgoipI1qptLFzUSiqL4U8+0v07wFzxqJBRFyZpi6XpqdOx+4KiRUJQAKBIdGTjaMlfUSCiKkjVqSkoXHSehKAFQLOGWVLigsgdrvtybUt16dFlKjlAjoSgBUh906q3nDQv8mGpMShcNNylKAJSqjizV61JSR42EoihZownu0kWNhFIUdGvTBIBBXTKbNrrQlGq4pT7lWpTcoDkJpSg4umcbnvvRmHprJBo6aktKFzUSStEwpFurQouQMaUabqkvV1Wqv38xoOEmRVHqPTrSOneokVCUACjVcItOFa6okVAURVF8USOhKAFQqg3pVGP92guqdFEjoSiKoviSlZEQkdtE5GMR+VBE/i0irR3brhWRVSKyQkQmOsonWWWrRGS6o7yPiCywyh8VkfJsZFOUvFKqDelUcxK5lUIpINl6Ei8BQ4wxw4CVwLUAIjIImAIMBiYBfxaRsIiEgbuBycAg4EKrLsCtwO3GmH7ADuDiLGVTFEVRsiQrI2GM+Y8xptr6Oh/obn0+C3jEGHPIGLMWWAWMsv5WGWPWGGOqgEeAsyQS0BwPPGHt/yBwdjayKUo+KdV++tq7SQkyJ/E9YLb1uRuw3rFtg1XmV94O2OkwOHa5oiiKUkCSjrgWkTlAZ49N1xljnrHqXAdUA/8IVjxfmaYB0wB69uyZj1MqSkJKtSWd6mWVqielpGAkjDETEm0Xkf8CzgROMcbYwx43Aj0c1bpbZfiUbwNai0iZ5U0463vJNAOYAVBZWalDLZWiQbuCFgY1Urkj295Nk4BrgK8ZY/Y7Ns0EpohIhYj0AfoD7wDvAv2tnkzlRJLbMy3j8gpwnrX/VOCZbGRTlHxSqioqVaOntrF0yXaCvz8BFcBL1sM03xhzqTFmqYg8BiwjEoa6zBhTAyAilwMvAmHgfmPMUutY/w94RERuAt4H/palbIqiKEqWZGUkrO6qfttuBm72KJ8FzPIoX0Ok95Oi1DvsFnddxLU0SD0nUVh0gr/coSOuFSUACq0kFSVXqJFQFMWXVHMN2o4vXdRIKEoAaOJWKVXUSCiK4kvKs8DmWA6lcKiRUJQA0H76SqmiRkJRAqTUBtOV2OUoGaBGQlGCQJWpUqKokVAUpd6j4b7coUZCUQJAwzJKqaJGQlEUX9T4KWokFCUAVJcWFp2WI3eokVAUxReN9StqJBQlAEq1HavhJkWNhKIoiuKLGglFUXxRR0JRI6EoiqL4okZCURRfSm2aESV91EgoiqIovqiRUBTFl/riR2hX3dyhRkJRFEXxRY2Eoii+aEpCUSOhKAGiOlUpNdRIKIriS7q9m9RIlh5lhRZAUUqB8nCkvTW6b7sCS1IYOrSoYNq4vpw3sntBzq8T/OWOrIyEiNwGfBWoAlYD3zXG7BSR3sByYIVVdb4x5lJrn5HAA0ATYBZwpTHGiEhb4FGgN7AO+IYxZkc28ilKvmjcKMycn4yjW+umhRalIIgIPz/9qEKLoeSAbMNNLwFDjDHDgJXAtY5tq40xw62/Sx3l9wCXAP2tv0lW+XRgrjGmPzDX+q4o9YZ+HVvQpDxcaDEUJVCyMhLGmP8YY6qtr/OBhL6miHQBWhpj5htjDPAQcLa1+SzgQevzg45yRVEUpUAEmbj+HjDb8b2PiLwvIq+JyFirrBuwwVFng1UG0MkYs8n6/AXQKUDZFEVRlAxImpMQkTlAZ49N1xljnrHqXAdUA/+wtm0Cehpjtlk5iKdFZHCqQlk5Ct9MlIhMA6YB9OzZM9XDKoqiKGmS1EgYYyYk2i4i/wWcCZxihZAwxhwCDlmfF4nIamAAsJHYkFR3qwxgs4h0McZsssJSWxLINAOYAVBZWandGhSlgaPTcuSOrMJNIjIJuAb4mjFmv6O8g4iErc99iSSo11jhpN0icqxEOmBfBDxj7TYTmGp9nuooVxRFUQpEtuMk/gRUAC9Zg27srq7jgF+LyGGgFrjUGLPd2ueH1HWBnU1dHuMW4DERuRj4FPhGlrIpiqIoWZKVkTDG9PMpfxJ40mfbQmCIR/k24JRs5FEURVGCRaflUBRFUXxRI6EoiqL4okZCURRF8UWNhKIo9R6d4C93qJFQFEVRfFEjoShKvUcH0+UONRKKoiiKL2okFEVRFF/USCiKoii+qJFQFEVRfFEjoShKHL/62mD6dmhWaDGUIiDbCf4URSlBph7fm6nH9y60GEoRoJ6EoiiK4osaCUVRFMUXNRKKoiiKL2okFEVRFF/USCiKoii+qJFQFKXec/2Zgzimdxsqe7UttCglh3aBVRSl3nNk5xY8funxhRajJFFPQlEURfFFjYSiKIriixoJRVEUxRc1EoqiKIovaiQURVEUX7I2EiLyGxH5UEQ+EJH/iEhXq1xE5C4RWWVtP9qxz1QR+cT6m+ooHykiH1n73CUiuiahoihKAQnCk7jNGDPMGDMceA64wSqfDPS3/qYB9wCISFvgl8BoYBTwSxFpY+1zD3CJY79JAcinKIqiZEjWRsIYs9vxtRlgrM9nAQ+ZCPOB1iLSBZgIvGSM2W6M2QG8BEyytrU0xsw3xhjgIeDsbOVTFEVRMieQwXQicjNwEbALONkq7gasd1TbYJUlKt/gUa4oiqIUiJSMhIjMATp7bLrOGPOMMeY64DoRuRa4nEg4KWeIyDQiISx69uyZy1MpSoPiH98fzZd7DxVaDKWISMlIGGMmpHi8fwCziBiJjUAPx7buVtlG4CRX+atWeXeP+l7yzABmAFRWVhqvOoqipM8J/doXWgSlyAiid1N/x9ezgI+tzzOBi6xeTscCu4wxm4AXgdNEpI2VsD4NeNHatltEjrV6NV0EPJOtfIqiKErmBJGTuEVEjgRqgU+BS63yWcDpwCpgP/BdAGPMdhH5DfCuVe/Xxpjt1ucfAg8ATYDZ1p+iKIpSICTSkaj+UllZaRYuXFhoMRRFUeoVIrLIGFOZrJ6OuFYURVF8USOhKIqi+KJGQlEURfFFjYSiKIriixoJRVEUxZd637tJRLYS6XqbCe2BLwMUpz6g19ww0GtuGGRzzb2MMR2SVar3RiIbRGRhKl3ASgm95oaBXnPDIB/XrOEmRVEUxRc1EoqiKIovDd1IzCi0AAVAr7lhoNfcMMj5NTfonISiKIqSmIbuSSiKoigJaLBGQkQmicgKEVklItMLLU8QiEgPEXlFRJaJyFIRudIqbysiL4nIJ9b/Nla5iMhd1m/woYgcXdgryBwRCYvI+yLynPW9j4gssK7tUREpt8orrO+rrO29Cyl3pohIaxF5QkQ+FpHlInJcqd9nEfmx9VwvEZF/iUjjUrvPInK/iGwRkSWOsrTvq4hMtep/IiJTs5GpQRoJEQkDdwOTgUHAhSIyqLBSBUI1cLUxZhBwLHCZdV3TgbnGmP7AXOs7RK6/v/U3Dbgn/yIHxpXAcsf3W4HbjTH9gB3AxVb5xcAOq/x2q1595E7gBWPMQOArRK69ZO+ziHQDrgAqjTFDgDAwhdK7zw8Ak1xlad1XEWlLZOG30cAo4Je2YckIY0yD+wOOI7LQkf39WuDaQsuVg+t8BjgVWAF0scq6ACusz/cBFzrqR+vVpz8iqxjOBcYDzwFCZIBRmft+E1n06jjrc5lVTwp9DWlebytgrVvuUr7PRNa7Xw+0te7bc8DEUrzPQG9gSab3FbgQuM9RHlMv3b8G6UlQ98DZbLDKSgbLvR4BLAA6mcjKfwBfAJ2sz6XyO9wBXENk4SuAdsBOY0y19d15XdFrtrbvsurXJ/oAW4H/s0JsfxWRZpTwfTbGbAT+F/gM2ETkvi2itO+zTbr3NdD73VCNREkjIs2BJ4GrjDG7ndtMpGlRMl3aRORMYIsxZlGhZckjZcDRwD3GmBHAPupCEEBJ3uc2RJZH7gN0BZoRH5YpeQpxXxuqkdgI9HB8726V1XtEpBERA/EPY8xTVvFmEelibe8CbLHKS+F3OAH4moisAx4hEnK6E2gtIvbyvM7ril6ztb0VsC2fAgfABmCDMWaB9f0JIkajlO/zBGCtMWarMeYw8BSRe1/K99km3fsa6P1uqEbiXaC/1TOinEgCbGaBZcoaERHgb8ByY8wfHJtmAnYPh6lEchV2+UVWL4ljgV0Ot7ZeYIy51hjT3RjTm8h9fNkY8y3gFeA8q5r7mu3f4jyrfr1qcRtjvgDWS2RteYBTgGWU8H0mEmY6VkSaWs+5fc0le58dpHtfXwROE5E2lgd2mlWWGYVO0hQwOXQ6sBJYDVxXaHkCuqYxRFzRD4EPrL/TicRi5wKfAHOAtlZ9IdLLazXwEZGeIwW/jiyu/yTgOetzX+AdYBXwOFBhlTe2vq+ytvcttNwZXutwYKF1r58G2pT6fQZ+BXwMLAH+DlSU2n0G/kUk53KYiMd4cSb3Ffiede2rgO9mI5OOuFYURVF8aajhJkVRFCUF1EgoiqIovqiRUBRFUXxRI6EoiqL4okZCURRF8UWNhKIoiuKLGglFURTFFzUSiqIoii//H1j6b5KZjESKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1053cf978>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(episode_rewards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAEICAYAAAC+iFRkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAF9FJREFUeJzt3X+wXGd93/H3pxY2vy3JOKqQ5LEZVKjDgLFvqRloxsUBbNeDnAnjMaW1bFzUtDRAYCaI0Damf2RCm4nBTccgfkUO5ochEGlUEnBk0t92kIEIgzGWCcYStmX8Q/wwY2L49o99LqyvZN/de/dqd+95v2Z29jnPeXb3Offc+9lnnz3n3FQVkqTl7e+NuwOSpKVn2EtSBxj2ktQBhr0kdYBhL0kdYNhLUgcY9pLUAYa9pkqSlyb5v0kOJbk/yf9J8o/aukuS/O+j3J9fTfKlJD9Ksj/JhX3rTktyU5KH2v1pR7NvUj/DXlMjydOBXcB/BVYD64B3Ag+PqT+nAh8F3gEcD7wAuKmtOxbYAXwEWAVsB3a0eumoM+w1Tf4BQFV9rKp+WlU/rqrPV9XeJP8QeC/w4iQ/TPIgQJLjkvxBku8kuSfJe5M8qa07q43GfyfJ95J8O8lrh+jPvwfeV1V/XlWPVNV9VXV7W3cWsAJ4d1U9XFVXAgFeNpofhTQcw17T5JvAT5NsT3JuklWzK6rqFuA3gP9XVU+tqpVt1e/Te5M4DXg2vU8D/7HvOf8+8IxWvxnYluQ5AEn+eZK9j9OfM1u7rya5K8lHkqxu634Z2FuPvh7J3lYvHXWGvaZGVX0feClQwPuBe5PsTLLmSO2TBNgC/FZV3V9VPwB+D7hoTtP/0Ebf/wP478CF7fU+WlXPf5wurQf+JfDrwEbgSfSmmACeChya0/4Q8LSBNlYasRXj7oA0jDaCvwQgyXPpzYm/G3jNEZqfCDwZuKmX+0BvKuWYvjYPVNWP+pbvAJ45YHd+DHy4qr7Z+vN7wF+2dT8Enj6n/dOBHwz43NJIObLX1KqqbwB/DDxvtmpOk+/RC+RfrqqV7XZ8VT21r82qJE/pWz4J+O6AXdg75zX7y18Dnp++dxng+a1eOuoMe02NJM9N8tYk69vyBnoj+htak3uA9bNHvFTVz+hN91yR5JfaY9YleeWcp35nkmOT/BPgfOCTA3bpw8ClSZ6V5MnAVnpHCwH8FfBT4I3tS+J/1+qvH26rpdEw7DVNfgD8Y+DGJD+iF/I3A29t66+nN3K+O8n3Wt3bgH3ADUm+T2+a5Tl9z3k38AC90fw1wG+0TwwkeW2SxxyJV9WHgKuBG+lN/zwMvLGt+wlwAXAx8CDwOuCCVi8ddfGfl6irkpwFfKSq1o+7L9JSc2QvSR2wJGGf5JwktybZl2TrUryGJGlwI5/GSXIMvZNfXg7sB74IvKaqvj7SF5IkDWwpRvYvAvZV1bfal1EfBzYtwetIkga0FCdVrQPu7FveT+8IikdJsoXe2Y0AZyxBPyRpqlVV5m81mLGdQVtV24BtAEk8JEiSltBSTOMcADb0La9vdZKkMVmKsP8isDHJKe1MxouAnUvwOpKkAY18GqeqHmmnhn+O3gWnPlRVXg9EksZoIs6gdc5ekg43yi9oPYNWkjrAsJekDjDsJakDDHtJ6gDDXpI6wLCXpA4w7CWpAwx7SeoAw16SOsCwl6QOMOwlqQMMe0nqAMNekjrAsJekDjDsJakDDHtJ6gDDXpI6wLCXpA4w7CWpA+YN+yQfSnIwyc19dauTXJfktna/qtUnyZVJ9iXZm+T0pey8JGkwg4zs/xg4Z07dVmB3VW0EdrdlgHOBje22BbhqNN2UJC3GvGFfVf8TuH9O9SZgeytvBy7oq7+6em4AViZZO6rOSpIWZqFz9muq6q5WvhtY08rrgDv72u1vdYdJsiXJniR7FtgHSdKAViz2CaqqktQCHrcN2AawkMdLkga30JH9PbPTM+3+YKs/AGzoa7e+1UmSxmihYb8T2NzKm4EdffUXt6NyzgQO9U33SJLGJFWPP4OS5GPAWcAzgHuA3wX+DLgWOAm4A7iwqu5PEuCP6B298xBwaVXNOyfvNI4kHa6qMqrnmjfsjwbDXpION8qw9wxaSeoAw16SOsCwl6QOMOwlqQMMe0nqAMNekjrAsJekDjDsJakDDHtJ6gDDXpI6wLCXpA4w7CWpAwx7SeoAw16SOsCwl6QOMOwlqQMMe0nqAMNekjpg3rBPsiHJF5J8PcnXkryp1a9Ocl2S29r9qlafJFcm2Zdkb5LTl3ojJEmPb5CR/SPAW6vqVOBM4A1JTgW2AruraiOwuy0DnAtsbLctwFUj77UkaSjzhn1V3VVVX2rlHwC3AOuATcD21mw7cEErbwKurp4bgJVJ1o6855KkgQ01Z5/kZOCFwI3Amqq6q626G1jTyuuAO/setr/VzX2uLUn2JNkzZJ8lSUMaOOyTPBX4U+DNVfX9/nVVVUAN88JVta2qZqpqZpjHSZKGN1DYJ3kCvaC/pqo+3arvmZ2eafcHW/0BYEPfw9e3OknSmAxyNE6ADwK3VNUf9q3aCWxu5c3Ajr76i9tROWcCh/qmeyRJY5DeDMzjNEheCvwv4KvAz1r179Cbt78WOAm4A7iwqu5vbw5/BJwDPARcWlWPOy+fZKgpIEnqgqrKqJ5r3rA/Ggx7STrcKMPeM2glqQMMe0nqAMNekjrAsJekDjDsJakDDHtJ6gDDXpI6wLCXpA4w7CWpAwx7SeoAw16SOsCwl6QOMOwlqQMMe0nqAMNekjrAsJekDjDsJakDDHtJ6oBB/uH4E5P8dZK/SfK1JO9s9ackuTHJviSfSHJsqz+uLe9r609e2k2QJM1nkJH9w8DLquoFwGnAOUnOBN4FXFFVzwYeAC5r7S8DHmj1V7R2kqQxmjfsq+eHbfEJ7VbAy4BPtfrtwAWtvKkt09afnWRk/zRXkjS8gebskxyT5CvAQeA64Hbgwap6pDXZD6xr5XXAnQBt/SHghCM855Yke5LsWdwmSJLmM1DYV9VPq+o0YD3wIuC5i33hqtpWVTNVNbPY55IkPb6hjsapqgeBLwAvBlYmWdFWrQcOtPIBYANAW388cN9IeitJWpBBjsY5McnKVn4S8HLgFnqh/+rWbDOwo5V3tmXa+uurqkbZaUnScDJfDid5Pr0vXI+h9+ZwbVX9pyTPAj4OrAa+DPyLqno4yROBPwFeCNwPXFRV35rnNXwzkKQ5qmpkB7fMG/ZHg2EvSYcbZdh7Bq0kdYBhL0kdYNhLUgcY9pLUAYa9JHWAYS9JHWDYS1IHGPaS1AGGvSR1gGEvSR1g2EtSBxj2ktQBhr0kdYBhL0kdYNhLUgcY9pLUAYa9JHWAYS9JHTBw2Cc5JsmXk+xqy6ckuTHJviSfSHJsqz+uLe9r609emq5LkgY1zMj+TcAtfcvvAq6oqmcDDwCXtfrLgAda/RWtnSRpjAYK+yTrgX8GfKAtB3gZ8KnWZDtwQStvasu09We39pKkMRl0ZP9u4LeBn7XlE4AHq+qRtrwfWNfK64A7Adr6Q639oyTZkmRPkj0L7LskaUDzhn2S84GDVXXTKF+4qrZV1UxVzYzyeSVJh1sxQJuXAK9Kch7wRODpwHuAlUlWtNH7euBAa38A2ADsT7ICOB64b+Q9lyQNbN6RfVW9varWV9XJwEXA9VX1WuALwKtbs83Ajlbe2ZZp66+vqhppryVJQ1nMcfZvA96SZB+9OfkPtvoPAie0+rcAWxfXRUnSYmUSBt1Jxt8JSZowVTWyIxk9g1aSOsCwl6QOMOwlqQMMe0nqAMNekjrAsJekDjDsJakDDHtJ6gDDXpI6wLCXpA4w7CWpAwx7SeoAw16SOsCwl6QOmIiwP+OMM5iESy1L0nI1EWE/q6oMfUlaAhMV9rMMfUkarYkM+1mGviSNxkBhn+TbSb6a5CtJ9rS61UmuS3Jbu1/V6pPkyiT7kuxNcvpiO2noS9LiDDOy/6dVdVpVzbTlrcDuqtoI7OYX/1j8XGBju20BrhpVZw19SVqYxUzjbAK2t/J24IK++qur5wZgZZK1i3idwxj6kjScQcO+gM8nuSnJlla3pqruauW7gTWtvA64s++x+1vdoyTZkmRPkj333nvvArqOga9lac+ecfdAy9GKAdu9tKoOJPkl4Lok3+hfWVWVZKjkraptwDaAmZmZBaf2bOAnWehTaII8VtDNzBy5frk60s+haz8DjdZAYV9VB9r9wSSfAV4E3JNkbVXd1aZpDrbmB4ANfQ9f3+qWlKG/vBl+vhFqceadxknylCRPmy0DrwBuBnYCm1uzzcCOVt4JXNyOyjkTONQ33bPknM+XpMMNMrJfA3ymjZhXAB+tqr9I8kXg2iSXAXcAF7b2nwXOA/YBDwGXjrzXA3Ckv7w4evVnoMWZN+yr6lvAC45Qfx9w9hHqC3jDSHo3Aob+dDHQevw56Iwzzhjp8030GbSj5PSOpoVBr6XQmbCfZehLmnRLkVGDHnq57Di9s7ycf/7lh9XNPPP1h9Xt+e772bXr8LbSJFjKgWhnw35WVRn4U6Y/2I8U6JIO1/mwh0e/mxr8k28xAT/zzNezi8tH1xlpRJZ6etmwn8Pgl3Q0Ha3vEDv3Be0w/DJ38hxpbn4czyFNG0f2A3C0L2kpHM3BpCP7Ic2O9h3xTy+/1NUkONoZYtgvgsE/vZzK0TiNIzOcxhkRp3okDWJcg0NH9kvA0f7kmzuV40hfR8M4c8GwX0JO80yHy7cccB5fS27cOWDYHyUG/8JcvuUAl285sKQj7z3ffT/g6F5LZxL+7g37MTD4J5Ojey2FSfk7N+zHzNAfzFJevKz/uR3da5Qm6W/bo3EmxNxfCo/oGX3wzk7XPNa6mWe+3mvnaGQmKejBsJ9Yj/WL0sU3gf6Q3rXrcma2vP6I6/oN+0mg/3nPP/9yL4OsRZm0oIcBwz7JSuADwPOAAl4H3Ap8AjgZ+DZwYVU9kF4avYfe/6F9CLikqr408p53VJc+ATzWHPpSXZPe0b1GYRKDHgafs38P8BdV9Vx6/4/2FmArsLuqNgK72zLAucDGdtsCXDXSHutR+r/sXa7z/3ODfalG3Y7mtViT/Pc378g+yfHArwCXAFTVT4CfJNkEnNWabQf+CngbsAm4uv3j8RuSrEyytqruGnnvdURH+oWbtk8As/P1jzfPvhQu37buqL6elo9JDnoYbBrnFOBe4MNJXgDcBLwJWNMX4HcDa1p5HXBn3+P3tzrDfoym7TuAXbsuh/PH3QtpMJMe9DBY2K8ATgd+s6puTPIefjFlA0BVVZKhtjbJFnrTPJx00knDPFQjNA2/pPC74+7AUTepb8I63HT8DQ02Z78f2F9VN7blT9EL/3uSrAVo9wfb+gPAhr7Hr291j1JV26pqpqpmTjzxxIX2X1qWluv3L8vNNO2jecO+qu4G7kzynFZ1NvB1YCewudVtBna08k7g4vScCRxyvl5aGEN/ck3bfhn0OPvfBK5JcizwLeBSem8U1ya5DLgDuLC1/Sy9wy730Tv08tKR9ljqoNlgcXpnMkxb0MOAYV9VXwFmjrDq7CO0LeANi+yXpCMw9MdvGoMevDaONJWc3hmPaf6ZG/bSFDP0j55p/zkb9tIyYOgvreXws/VCaNIyMu1z+gsJ1aXe1uUQ9GDYS8vSNIX+YsN0kMcv9OewXIIeDHtpWZvE0B9HgC6n0F4ow17qgHGFviE7OQx7qUOqyjnujjLspY7pD+NRBb8BP/kMe6nDFjq9Y7hPH8Ne0kDTOwb8dDPsJQGG+XLnGbSS1AGGvSR1gGEvSR1g2EtSBxj2ktQBhr0kdYBhL0kdMG/YJ3lOkq/03b6f5M1JVie5Lslt7X5Va58kVybZl2RvktOXfjMkSY9n3rCvqlur6rSqOg04A3gI+AywFdhdVRuB3W0Z4FxgY7ttAa5aio5LkgY37DTO2cDtVXUHsAnY3uq3Axe08ibg6uq5AViZZO1IeitJWpBhw/4i4GOtvKaq7mrlu4E1rbwOuLPvMftb3aMk2ZJkT5I9995775DdkCQNY+CwT3Is8Crgk3PXVe+iGkNdWKOqtlXVTFXNnHjiicM8VJI0pGFG9ucCX6qqe9ryPbPTM+3+YKs/AGzoe9z6VidJGpNhwv41/GIKB2AnsLmVNwM7+uovbkflnAkc6pvukSSNwUCXOE7yFODlwL/uq/594NoklwF3ABe2+s8C5wH76B25c+nIeitJWpCBwr6qfgScMKfuPnpH58xtW8AbRtI7SdJIeAatJHWAYS9JHWDYS1IHGPaS1AGGvSR1gGEvSR1g2EtSBxj2ktQBhr0kdYBhL0kdYNhLUgcY9pLUAYa9JHWAYS9JHWDYS1IHGPaS1AGGvSR1gGEvSR0wUNgn+a0kX0tyc5KPJXliklOS3JhkX5JPJDm2tT2uLe9r609eyg2QJM1v3rBPsg54IzBTVc8DjgEuAt4FXFFVzwYeAC5rD7kMeKDVX9HaSZLGaNBpnBXAk5KsAJ4M3AW8DPhUW78duKCVN7Vl2vqzk2Q03ZUkLcSK+RpU1YEkfwB8B/gx8HngJuDBqnqkNdsPrGvldcCd7bGPJDkEnAB8r/95k2wBtrTFh5PcvMhtmVTPYM62LxNu1/RZrtu2XLfrOaN8snnDPskqeqP1U4AHgU8C5yz2hatqG7CtvcaeqppZ7HNOouW6bW7X9Fmu27act2uUzzfINM6vAn9bVfdW1d8BnwZeAqxs0zoA64EDrXwA2NA6uwI4HrhvlJ2WJA1nkLD/DnBmkie3ufezga8DXwBe3dpsBna08s62TFt/fVXV6LosSRrWvGFfVTfS+6L1S8BX22O2AW8D3pJkH705+Q+2h3wQOKHVvwXYOkA/tg3f9amxXLfN7Zo+y3Xb3K4BxEG3JC1/nkErSR1g2EtSB4w97JOck+TWdnmFQeb3J0aSDUm+kOTr7XISb2r1q5Ncl+S2dr+q1SfJlW1b9yY5fbxb8PiSHJPky0l2teVlcYmMJCuTfCrJN5LckuTFy2GfLafLmiT5UJKD/effLGQfJdnc2t+WZPORXutoeozt+i/td3Fvks8kWdm37u1tu25N8sq++uFzs6rGdqN36YXbgWcBxwJ/A5w6zj4N2f+1wOmt/DTgm8CpwH8Gtrb6rcC7Wvk84M+BAGcCN457G+bZvrcAHwV2teVrgYta+b3Av2nlfwu8t5UvAj4x7r7Ps13bgX/VyscCK6d9n9E7mfFvgSf17atLpnWfAb8CnA7c3Fc31D4CVgPfaverWnnVBG7XK4AVrfyuvu06tWXicfTOc7q9ZeaCcnPcO/TFwOf6lt8OvH3cv2iL2J4dwMuBW4G1rW4tcGsrvw94TV/7n7ebtBu9cyd207ssxq72h/S9vl/Kn+874HPAi1t5RWuXcW/DY2zX8S0UM6d+qvcZvzhzfXXbB7uAV07zPgNOnhOKQ+0j4DXA+/rqH9VuUrZrzrpfA65p5Ufl4ew+W2hujnsa5+eXVmj6L7swVdrH4BcCNwJrququtupuYE0rT9P2vhv4beBnbfkEBrxEBjB7iYxJdApwL/DhNkX1gSRPYcr3WVUdAGYva3IXvX0w8GVNmOx9NmvYfTQV+26O19H7lAIj3q5xh/2ykOSpwJ8Cb66q7/evq95b71Qd35rkfOBgVd007r4sgRX0PkZfVVUvBH7EnHNBpnSf9V/W5JnAUxjBZU0m1TTuo/kkeQfwCHDNUjz/uMP+55dWaPovuzAVkjyBXtBfU1WfbtX3JFnb1q8FDrb6adnelwCvSvJt4OP0pnLew/K4RMZ+YH/1ThaE3gmDpzP9+6wLlzUZdh9Ny74jySXA+cBr2xsZjHi7xh32XwQ2tiMGjqX3RdHOMfdpYElC74zhW6rqD/tW9V8yYu6lJC5uRw+cCRzq+1g6Marq7VW1vqpOprdPrq+q17IMLpFRVXcDdyaZvaLg7OU/pnqf0Y3Lmgy7jz4HvCLJqvbJ5xWtbqIkOYfelOmrquqhvlU7gYvakVOnABuBv2ahuTkBX1acR+8oltuBd4y7P0P2/aX0PkruBb7SbufRm/vcDdwG/CWwurUP8N/atn6V3j+EGft2zLONZ/GLo3Ge1X7Z9tG7+ulxrf6JbXlfW/+scfd7nm06DdjT9tuf0TtSY+r3GfBO4BvAzcCf0DuKYyr3GfAxet89/B29T2OXLWQf0ZsD39dul07odu2jNwc/myHv7Wv/jrZdtwLn9tUPnZteLkGSOmDc0ziSpKPAsJekDjDsJakDDHtJ6gDDXpI6wLCXpA4w7CWpA/4/f4RKXVZOo2kAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1053ab860>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAEICAYAAAC+iFRkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAF9FJREFUeJzt3X+wXGd93/H3pxY2vy3JOKqQ5LEZVKjDgLFvqRloxsUBbNeDnAnjMaW1bFzUtDRAYCaI0Damf2RCm4nBTccgfkUO5ochEGlUEnBk0t92kIEIgzGWCcYStmX8Q/wwY2L49o99LqyvZN/de/dqd+95v2Z29jnPeXb3Offc+9lnnz3n3FQVkqTl7e+NuwOSpKVn2EtSBxj2ktQBhr0kdYBhL0kdYNhLUgcY9pLUAYa9pkqSlyb5v0kOJbk/yf9J8o/aukuS/O+j3J9fTfKlJD9Ksj/JhX3rTktyU5KH2v1pR7NvUj/DXlMjydOBXcB/BVYD64B3Ag+PqT+nAh8F3gEcD7wAuKmtOxbYAXwEWAVsB3a0eumoM+w1Tf4BQFV9rKp+WlU/rqrPV9XeJP8QeC/w4iQ/TPIgQJLjkvxBku8kuSfJe5M8qa07q43GfyfJ95J8O8lrh+jPvwfeV1V/XlWPVNV9VXV7W3cWsAJ4d1U9XFVXAgFeNpofhTQcw17T5JvAT5NsT3JuklWzK6rqFuA3gP9XVU+tqpVt1e/Te5M4DXg2vU8D/7HvOf8+8IxWvxnYluQ5AEn+eZK9j9OfM1u7rya5K8lHkqxu634Z2FuPvh7J3lYvHXWGvaZGVX0feClQwPuBe5PsTLLmSO2TBNgC/FZV3V9VPwB+D7hoTtP/0Ebf/wP478CF7fU+WlXPf5wurQf+JfDrwEbgSfSmmACeChya0/4Q8LSBNlYasRXj7oA0jDaCvwQgyXPpzYm/G3jNEZqfCDwZuKmX+0BvKuWYvjYPVNWP+pbvAJ45YHd+DHy4qr7Z+vN7wF+2dT8Enj6n/dOBHwz43NJIObLX1KqqbwB/DDxvtmpOk+/RC+RfrqqV7XZ8VT21r82qJE/pWz4J+O6AXdg75zX7y18Dnp++dxng+a1eOuoMe02NJM9N8tYk69vyBnoj+htak3uA9bNHvFTVz+hN91yR5JfaY9YleeWcp35nkmOT/BPgfOCTA3bpw8ClSZ6V5MnAVnpHCwH8FfBT4I3tS+J/1+qvH26rpdEw7DVNfgD8Y+DGJD+iF/I3A29t66+nN3K+O8n3Wt3bgH3ADUm+T2+a5Tl9z3k38AC90fw1wG+0TwwkeW2SxxyJV9WHgKuBG+lN/zwMvLGt+wlwAXAx8CDwOuCCVi8ddfGfl6irkpwFfKSq1o+7L9JSc2QvSR2wJGGf5JwktybZl2TrUryGJGlwI5/GSXIMvZNfXg7sB74IvKaqvj7SF5IkDWwpRvYvAvZV1bfal1EfBzYtwetIkga0FCdVrQPu7FveT+8IikdJsoXe2Y0AZyxBPyRpqlVV5m81mLGdQVtV24BtAEk8JEiSltBSTOMcADb0La9vdZKkMVmKsP8isDHJKe1MxouAnUvwOpKkAY18GqeqHmmnhn+O3gWnPlRVXg9EksZoIs6gdc5ekg43yi9oPYNWkjrAsJekDjDsJakDDHtJ6gDDXpI6wLCXpA4w7CWpAwx7SeoAw16SOsCwl6QOMOwlqQMMe0nqAMNekjrAsJekDjDsJakDDHtJ6gDDXpI6wLCXpA4w7CWpA+YN+yQfSnIwyc19dauTXJfktna/qtUnyZVJ9iXZm+T0pey8JGkwg4zs/xg4Z07dVmB3VW0EdrdlgHOBje22BbhqNN2UJC3GvGFfVf8TuH9O9SZgeytvBy7oq7+6em4AViZZO6rOSpIWZqFz9muq6q5WvhtY08rrgDv72u1vdYdJsiXJniR7FtgHSdKAViz2CaqqktQCHrcN2AawkMdLkga30JH9PbPTM+3+YKs/AGzoa7e+1UmSxmihYb8T2NzKm4EdffUXt6NyzgQO9U33SJLGJFWPP4OS5GPAWcAzgHuA3wX+DLgWOAm4A7iwqu5PEuCP6B298xBwaVXNOyfvNI4kHa6qMqrnmjfsjwbDXpION8qw9wxaSeoAw16SOsCwl6QOMOwlqQMMe0nqAMNekjrAsJekDjDsJakDDHtJ6gDDXpI6wLCXpA4w7CWpAwx7SeoAw16SOsCwl6QOMOwlqQMMe0nqAMNekjpg3rBPsiHJF5J8PcnXkryp1a9Ocl2S29r9qlafJFcm2Zdkb5LTl3ojJEmPb5CR/SPAW6vqVOBM4A1JTgW2AruraiOwuy0DnAtsbLctwFUj77UkaSjzhn1V3VVVX2rlHwC3AOuATcD21mw7cEErbwKurp4bgJVJ1o6855KkgQ01Z5/kZOCFwI3Amqq6q626G1jTyuuAO/setr/VzX2uLUn2JNkzZJ8lSUMaOOyTPBX4U+DNVfX9/nVVVUAN88JVta2qZqpqZpjHSZKGN1DYJ3kCvaC/pqo+3arvmZ2eafcHW/0BYEPfw9e3OknSmAxyNE6ADwK3VNUf9q3aCWxu5c3Ajr76i9tROWcCh/qmeyRJY5DeDMzjNEheCvwv4KvAz1r179Cbt78WOAm4A7iwqu5vbw5/BJwDPARcWlWPOy+fZKgpIEnqgqrKqJ5r3rA/Ggx7STrcKMPeM2glqQMMe0nqAMNekjrAsJekDjDsJakDDHtJ6gDDXpI6wLCXpA4w7CWpAwx7SeoAw16SOsCwl6QOMOwlqQMMe0nqAMNekjrAsJekDjDsJakDDHtJ6oBB/uH4E5P8dZK/SfK1JO9s9ackuTHJviSfSHJsqz+uLe9r609e2k2QJM1nkJH9w8DLquoFwGnAOUnOBN4FXFFVzwYeAC5r7S8DHmj1V7R2kqQxmjfsq+eHbfEJ7VbAy4BPtfrtwAWtvKkt09afnWRk/zRXkjS8gebskxyT5CvAQeA64Hbgwap6pDXZD6xr5XXAnQBt/SHghCM855Yke5LsWdwmSJLmM1DYV9VPq+o0YD3wIuC5i33hqtpWVTNVNbPY55IkPb6hjsapqgeBLwAvBlYmWdFWrQcOtPIBYANAW388cN9IeitJWpBBjsY5McnKVn4S8HLgFnqh/+rWbDOwo5V3tmXa+uurqkbZaUnScDJfDid5Pr0vXI+h9+ZwbVX9pyTPAj4OrAa+DPyLqno4yROBPwFeCNwPXFRV35rnNXwzkKQ5qmpkB7fMG/ZHg2EvSYcbZdh7Bq0kdYBhL0kdYNhLUgcY9pLUAYa9JHWAYS9JHWDYS1IHGPaS1AGGvSR1gGEvSR1g2EtSBxj2ktQBhr0kdYBhL0kdYNhLUgcY9pLUAYa9JHWAYS9JHTBw2Cc5JsmXk+xqy6ckuTHJviSfSHJsqz+uLe9r609emq5LkgY1zMj+TcAtfcvvAq6oqmcDDwCXtfrLgAda/RWtnSRpjAYK+yTrgX8GfKAtB3gZ8KnWZDtwQStvasu09We39pKkMRl0ZP9u4LeBn7XlE4AHq+qRtrwfWNfK64A7Adr6Q639oyTZkmRPkj0L7LskaUDzhn2S84GDVXXTKF+4qrZV1UxVzYzyeSVJh1sxQJuXAK9Kch7wRODpwHuAlUlWtNH7euBAa38A2ADsT7ICOB64b+Q9lyQNbN6RfVW9varWV9XJwEXA9VX1WuALwKtbs83Ajlbe2ZZp66+vqhppryVJQ1nMcfZvA96SZB+9OfkPtvoPAie0+rcAWxfXRUnSYmUSBt1Jxt8JSZowVTWyIxk9g1aSOsCwl6QOMOwlqQMMe0nqAMNekjrAsJekDjDsJakDDHtJ6gDDXpI6wLCXpA4w7CWpAwx7SeoAw16SOsCwl6QOmIiwP+OMM5iESy1L0nI1EWE/q6oMfUlaAhMV9rMMfUkarYkM+1mGviSNxkBhn+TbSb6a5CtJ9rS61UmuS3Jbu1/V6pPkyiT7kuxNcvpiO2noS9LiDDOy/6dVdVpVzbTlrcDuqtoI7OYX/1j8XGBju20BrhpVZw19SVqYxUzjbAK2t/J24IK++qur5wZgZZK1i3idwxj6kjScQcO+gM8nuSnJlla3pqruauW7gTWtvA64s++x+1vdoyTZkmRPkj333nvvArqOga9lac+ecfdAy9GKAdu9tKoOJPkl4Lok3+hfWVWVZKjkraptwDaAmZmZBaf2bOAnWehTaII8VtDNzBy5frk60s+haz8DjdZAYV9VB9r9wSSfAV4E3JNkbVXd1aZpDrbmB4ANfQ9f3+qWlKG/vBl+vhFqceadxknylCRPmy0DrwBuBnYCm1uzzcCOVt4JXNyOyjkTONQ33bPknM+XpMMNMrJfA3ymjZhXAB+tqr9I8kXg2iSXAXcAF7b2nwXOA/YBDwGXjrzXA3Ckv7w4evVnoMWZN+yr6lvAC45Qfx9w9hHqC3jDSHo3Aob+dDHQevw56Iwzzhjp8030GbSj5PSOpoVBr6XQmbCfZehLmnRLkVGDHnq57Di9s7ycf/7lh9XNPPP1h9Xt+e772bXr8LbSJFjKgWhnw35WVRn4U6Y/2I8U6JIO1/mwh0e/mxr8k28xAT/zzNezi8tH1xlpRJZ6etmwn8Pgl3Q0Ha3vEDv3Be0w/DJ38hxpbn4czyFNG0f2A3C0L2kpHM3BpCP7Ic2O9h3xTy+/1NUkONoZYtgvgsE/vZzK0TiNIzOcxhkRp3okDWJcg0NH9kvA0f7kmzuV40hfR8M4c8GwX0JO80yHy7cccB5fS27cOWDYHyUG/8JcvuUAl285sKQj7z3ffT/g6F5LZxL+7g37MTD4J5Ojey2FSfk7N+zHzNAfzFJevKz/uR3da5Qm6W/bo3EmxNxfCo/oGX3wzk7XPNa6mWe+3mvnaGQmKejBsJ9Yj/WL0sU3gf6Q3rXrcma2vP6I6/oN+0mg/3nPP/9yL4OsRZm0oIcBwz7JSuADwPOAAl4H3Ap8AjgZ+DZwYVU9kF4avYfe/6F9CLikqr408p53VJc+ATzWHPpSXZPe0b1GYRKDHgafs38P8BdV9Vx6/4/2FmArsLuqNgK72zLAucDGdtsCXDXSHutR+r/sXa7z/3ODfalG3Y7mtViT/Pc378g+yfHArwCXAFTVT4CfJNkEnNWabQf+CngbsAm4uv3j8RuSrEyytqruGnnvdURH+oWbtk8As/P1jzfPvhQu37buqL6elo9JDnoYbBrnFOBe4MNJXgDcBLwJWNMX4HcDa1p5HXBn3+P3tzrDfoym7TuAXbsuh/PH3QtpMJMe9DBY2K8ATgd+s6puTPIefjFlA0BVVZKhtjbJFnrTPJx00knDPFQjNA2/pPC74+7AUTepb8I63HT8DQ02Z78f2F9VN7blT9EL/3uSrAVo9wfb+gPAhr7Hr291j1JV26pqpqpmTjzxxIX2X1qWluv3L8vNNO2jecO+qu4G7kzynFZ1NvB1YCewudVtBna08k7g4vScCRxyvl5aGEN/ck3bfhn0OPvfBK5JcizwLeBSem8U1ya5DLgDuLC1/Sy9wy730Tv08tKR9ljqoNlgcXpnMkxb0MOAYV9VXwFmjrDq7CO0LeANi+yXpCMw9MdvGoMevDaONJWc3hmPaf6ZG/bSFDP0j55p/zkb9tIyYOgvreXws/VCaNIyMu1z+gsJ1aXe1uUQ9GDYS8vSNIX+YsN0kMcv9OewXIIeDHtpWZvE0B9HgC6n0F4ow17qgHGFviE7OQx7qUOqyjnujjLspY7pD+NRBb8BP/kMe6nDFjq9Y7hPH8Ne0kDTOwb8dDPsJQGG+XLnGbSS1AGGvSR1gGEvSR1g2EtSBxj2ktQBhr0kdYBhL0kdMG/YJ3lOkq/03b6f5M1JVie5Lslt7X5Va58kVybZl2RvktOXfjMkSY9n3rCvqlur6rSqOg04A3gI+AywFdhdVRuB3W0Z4FxgY7ttAa5aio5LkgY37DTO2cDtVXUHsAnY3uq3Axe08ibg6uq5AViZZO1IeitJWpBhw/4i4GOtvKaq7mrlu4E1rbwOuLPvMftb3aMk2ZJkT5I9995775DdkCQNY+CwT3Is8Crgk3PXVe+iGkNdWKOqtlXVTFXNnHjiicM8VJI0pGFG9ucCX6qqe9ryPbPTM+3+YKs/AGzoe9z6VidJGpNhwv41/GIKB2AnsLmVNwM7+uovbkflnAkc6pvukSSNwUCXOE7yFODlwL/uq/594NoklwF3ABe2+s8C5wH76B25c+nIeitJWpCBwr6qfgScMKfuPnpH58xtW8AbRtI7SdJIeAatJHWAYS9JHWDYS1IHGPaS1AGGvSR1gGEvSR1g2EtSBxj2ktQBhr0kdYBhL0kdYNhLUgcY9pLUAYa9JHWAYS9JHWDYS1IHGPaS1AGGvSR1gGEvSR0wUNgn+a0kX0tyc5KPJXliklOS3JhkX5JPJDm2tT2uLe9r609eyg2QJM1v3rBPsg54IzBTVc8DjgEuAt4FXFFVzwYeAC5rD7kMeKDVX9HaSZLGaNBpnBXAk5KsAJ4M3AW8DPhUW78duKCVN7Vl2vqzk2Q03ZUkLcSK+RpU1YEkfwB8B/gx8HngJuDBqnqkNdsPrGvldcCd7bGPJDkEnAB8r/95k2wBtrTFh5PcvMhtmVTPYM62LxNu1/RZrtu2XLfrOaN8snnDPskqeqP1U4AHgU8C5yz2hatqG7CtvcaeqppZ7HNOouW6bW7X9Fmu27act2uUzzfINM6vAn9bVfdW1d8BnwZeAqxs0zoA64EDrXwA2NA6uwI4HrhvlJ2WJA1nkLD/DnBmkie3ufezga8DXwBe3dpsBna08s62TFt/fVXV6LosSRrWvGFfVTfS+6L1S8BX22O2AW8D3pJkH705+Q+2h3wQOKHVvwXYOkA/tg3f9amxXLfN7Zo+y3Xb3K4BxEG3JC1/nkErSR1g2EtSB4w97JOck+TWdnmFQeb3J0aSDUm+kOTr7XISb2r1q5Ncl+S2dr+q1SfJlW1b9yY5fbxb8PiSHJPky0l2teVlcYmMJCuTfCrJN5LckuTFy2GfLafLmiT5UJKD/effLGQfJdnc2t+WZPORXutoeozt+i/td3Fvks8kWdm37u1tu25N8sq++uFzs6rGdqN36YXbgWcBxwJ/A5w6zj4N2f+1wOmt/DTgm8CpwH8Gtrb6rcC7Wvk84M+BAGcCN457G+bZvrcAHwV2teVrgYta+b3Av2nlfwu8t5UvAj4x7r7Ps13bgX/VyscCK6d9n9E7mfFvgSf17atLpnWfAb8CnA7c3Fc31D4CVgPfaverWnnVBG7XK4AVrfyuvu06tWXicfTOc7q9ZeaCcnPcO/TFwOf6lt8OvH3cv2iL2J4dwMuBW4G1rW4tcGsrvw94TV/7n7ebtBu9cyd207ssxq72h/S9vl/Kn+874HPAi1t5RWuXcW/DY2zX8S0UM6d+qvcZvzhzfXXbB7uAV07zPgNOnhOKQ+0j4DXA+/rqH9VuUrZrzrpfA65p5Ufl4ew+W2hujnsa5+eXVmj6L7swVdrH4BcCNwJrququtupuYE0rT9P2vhv4beBnbfkEBrxEBjB7iYxJdApwL/DhNkX1gSRPYcr3WVUdAGYva3IXvX0w8GVNmOx9NmvYfTQV+26O19H7lAIj3q5xh/2ykOSpwJ8Cb66q7/evq95b71Qd35rkfOBgVd007r4sgRX0PkZfVVUvBH7EnHNBpnSf9V/W5JnAUxjBZU0m1TTuo/kkeQfwCHDNUjz/uMP+55dWaPovuzAVkjyBXtBfU1WfbtX3JFnb1q8FDrb6adnelwCvSvJt4OP0pnLew/K4RMZ+YH/1ThaE3gmDpzP9+6wLlzUZdh9Ny74jySXA+cBr2xsZjHi7xh32XwQ2tiMGjqX3RdHOMfdpYElC74zhW6rqD/tW9V8yYu6lJC5uRw+cCRzq+1g6Marq7VW1vqpOprdPrq+q17IMLpFRVXcDdyaZvaLg7OU/pnqf0Y3Lmgy7jz4HvCLJqvbJ5xWtbqIkOYfelOmrquqhvlU7gYvakVOnABuBv2ahuTkBX1acR+8oltuBd4y7P0P2/aX0PkruBb7SbufRm/vcDdwG/CWwurUP8N/atn6V3j+EGft2zLONZ/GLo3Ge1X7Z9tG7+ulxrf6JbXlfW/+scfd7nm06DdjT9tuf0TtSY+r3GfBO4BvAzcCf0DuKYyr3GfAxet89/B29T2OXLWQf0ZsD39dul07odu2jNwc/myHv7Wv/jrZdtwLn9tUPnZteLkGSOmDc0ziSpKPAsJekDjDsJakDDHtJ6gDDXpI6wLCXpA4w7CWpA/4/f4RKXVZOo2kAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1053ab860>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def show_state(env, step=0):\n",
    "    plt.figure(3)\n",
    "    plt.clf()\n",
    "    plt.imshow(env.render(mode='rgb_array'))\n",
    "    plt.title(\"Step: %d\" % (step))\n",
    "    \n",
    "    plt.pause(0.001)  # pause for plots to update\n",
    "    \n",
    "    display.clear_output(wait=True)\n",
    "    display.display(plt.gcf())\n",
    "        \n",
    "done = False\n",
    "state = env.reset()\n",
    "step = 0\n",
    "while not done:\n",
    "    step += 1\n",
    "    show_state(env, step)\n",
    "    action = policy(w_p, state)\n",
    "    state, _, done, _ = env.step(action)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}