Chrome_Dino_RL_Project.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ef31ac84-e675-4fab-8909-3e9bf7b234f9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: stable-baselines[extra] in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (2.10.2)\n",
      "\u001b[33mWARNING: stable-baselines 2.10.2 does not provide the extra 'extra'\u001b[0m\u001b[33m\n",
      "\u001b[0mRequirement already satisfied: joblib in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from stable-baselines[extra]) (1.2.0)\n",
      "Requirement already satisfied: opencv-python in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from stable-baselines[extra]) (4.8.1.78)\n",
      "Requirement already satisfied: gym[atari,classic_control]>=0.11 in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from stable-baselines[extra]) (0.23.1)\n",
      "Requirement already satisfied: cloudpickle>=0.5.5 in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from stable-baselines[extra]) (3.0.0)\n",
      "Requirement already satisfied: scipy in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from stable-baselines[extra]) (1.10.1)\n",
      "Requirement already satisfied: pandas in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from stable-baselines[extra]) (1.5.3)\n",
      "Requirement already satisfied: numpy in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from stable-baselines[extra]) (1.26.0)\n",
      "Requirement already satisfied: matplotlib in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from stable-baselines[extra]) (3.8.2)\n",
      "Requirement already satisfied: gym-notices>=0.0.4 in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from gym[atari,classic_control]>=0.11->stable-baselines[extra]) (0.0.8)\n",
      "Requirement already satisfied: pygame==2.1.0 in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from gym[atari,classic_control]>=0.11->stable-baselines[extra]) (2.1.0)\n",
      "Requirement already satisfied: ale-py~=0.7.4 in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from gym[atari,classic_control]>=0.11->stable-baselines[extra]) (0.7.5)\n",
      "Requirement already satisfied: pillow>=8 in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from matplotlib->stable-baselines[extra]) (10.1.0)\n",
      "Requirement already satisfied: contourpy>=1.0.1 in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from matplotlib->stable-baselines[extra]) (1.2.0)\n",
      "Requirement already satisfied: kiwisolver>=1.3.1 in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from matplotlib->stable-baselines[extra]) (1.4.5)\n",
      "Requirement already satisfied: fonttools>=4.22.0 in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from matplotlib->stable-baselines[extra]) (4.45.0)\n",
      "Requirement already satisfied: pyparsing>=2.3.1 in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from matplotlib->stable-baselines[extra]) (3.1.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from matplotlib->stable-baselines[extra]) (0.12.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from matplotlib->stable-baselines[extra]) (2.8.2)\n",
      "Requirement already satisfied: packaging>=20.0 in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from matplotlib->stable-baselines[extra]) (23.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from pandas->stable-baselines[extra]) (2023.3.post1)\n",
      "Requirement already satisfied: importlib-resources in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from ale-py~=0.7.4->gym[atari,classic_control]>=0.11->stable-baselines[extra]) (6.1.1)\n",
      "Requirement already satisfied: six>=1.5 in /opt/homebrew/Caskroom/miniforge/base/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib->stable-baselines[extra]) (1.16.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install 'stable-baselines[extra]' "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb657913-ec9b-4c11-9ba3-64a6a775151a",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install PyAutoGUI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "13555f73-fd61-463d-a4a2-2c9eb0d5fe00",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install mss pydirectinput pytesseract"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f9aa493-a446-4844-a3fe-c307de3a657b",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install opencv-python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "cf90b1c9-cdd1-4a59-988e-342f70bb9078",
   "metadata": {},
   "outputs": [],
   "source": [
    "from mss import mss\n",
    "#import pydirectinput\n",
    "import pyautogui\n",
    "import cv2\n",
    "import numpy as np\n",
    "import pytesseract\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "#from gym import Env\n",
    "from gymnasium import Env\n",
    "from gymnasium.spaces import Box, Discrete\n",
    "from DinoEnv import DinoGame"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "22aa25f6-7eb1-4620-82e4-a2f679dfa29e",
   "metadata": {},
   "outputs": [],
   "source": [
    "env = DinoGame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "f47d27df-8561-475a-b197-9fc567a421a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "obs = env.get_observation()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "275f5a43-826c-49b2-909b-700afdceaefc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAewAAAGgCAYAAABosFR3AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAjrElEQVR4nO3df3RU9Z3/8deQyJDQZLaozGRqwGDH9Ue0CrjRgCauJj3I8ejhrK2CFg/n9AsbaEk522BKVyNfnZF0N4fdpmLh9Iv4pVncnqJ1e6omtDXgpi4RN5aNHtQ1xViZprrpTBScbJLP9w+/3HUyQZjJDMkn83ycc89xPvdz77zzMeSVT+7n3nEZY4wAAMCkNm2iCwAAAKdHYAMAYAECGwAACxDYAABYgMAGAMACBDYAABYgsAEAsACBDQCABQhsAAAsQGADAGCBjAX2o48+qpKSEs2YMUMLFizQgQMHMvVWAABMebmZOOmTTz6p2tpaPfroo1q0aJF++MMfasmSJXrttdc0Z86czzx2ZGRE7733ngoKCuRyuTJRHgAAk4YxRgMDA/L7/Zo27dTzaFcmPvyjrKxM8+fP17Zt25y2Sy+9VLfffrtCodBnHvvuu++quLg43SUBADCp9fb26oILLjjl/rTPsAcHB3Xo0CHdd999ce3V1dXq6OhI6B+LxRSLxZzXJ39/uK58o3Jz3ekuDwCASWVoKKbfdGxRQUHBZ/ZLe2C///77Gh4eltfrjWv3er0Kh8MJ/UOhkB588MHEwnLdys2dke7yAACYlE53GThji85Gv7ExZsxi6uvrFYlEnK23tzdTJQEAYK20z7DPO+885eTkJMym+/r6EmbdkuR2u+V286dvAAA+S9pn2NOnT9eCBQvU1tYW197W1qby8vJ0vx0AAFkhI7d1bdiwQffcc48WLlyo6667Ttu3b9c777yjNWvWZOLtAACY8jIS2F/96lf1wQcfaPPmzTp27JhKS0v1i1/8QnPnzs3E2wEAMOVlJLAlqaamRjU1NZk6PQAAWYVniQMAYAECGwAACxDYAABYgMAGAMACBDYAABYgsAEAsACBDQCABQhsAAAsQGADAGABAhsAAAsQ2AAAWIDABgDAAgQ2AAAWILABALAAgQ0AgAUIbAAALEBgAwBgAQIbAAALENgAAFiAwAYAwAIENgAAFiCwAQCwAIENAIAFCGwAACxAYAMAYAECGwAACxDYAABYgMAGAMACBDYAABYgsAEAsACBDQCABQhsAAAsQGADAGABAhsAAAskHdj79+/XrbfeKr/fL5fLpaeffjpuvzFGDQ0N8vv9ysvLU2Vlpbq7u9NVLwAAWSnpwP7oo4/0pS99Sc3NzWPub2xsVFNTk5qbm9XZ2Smfz6eqqioNDAyMu1icmWF3TsJmprniNgCAXXKTPWDJkiVasmTJmPuMMdq6das2bdqkZcuWSZJ27dolr9erlpYWrV69OuGYWCymWCzmvI5Go8mWBADAlJfWa9g9PT0Kh8Oqrq522txutyoqKtTR0THmMaFQSB6Px9mKi4vTWRIAAFNCWgM7HA5Lkrxeb1y71+t19o1WX1+vSCTibL29veksCQCAKSHpP4mfCZcr/hqpMSah7SS32y23252JMrJWTmw4oW3YnZPScQCAySGtM2yfzydJCbPpvr6+hFk3AAA4c2kN7JKSEvl8PrW1tTltg4ODam9vV3l5eTrfCgCArJL0n8Q//PBDvfXWW87rnp4edXV1adasWZozZ45qa2sVDAYVCAQUCAQUDAaVn5+v5cuXp7VwAACySdKB/fLLL+vGG290Xm/YsEGStHLlSj3++OOqq6vTiRMnVFNTo/7+fpWVlam1tVUFBQXpqxpJG319eqxr2qPbuKYNAJOHyxhjJrqIT4tGo/J4PLr+hvuVmztjosuZsliEBgCTw9DQxzqwf7MikYgKCwtP2Y9niQMAYAECGwAACxDYAABYICMPTsHUMNaHhLhGJtWSBwDIGsywAQCwAIENAIAFCGwAACzANWyc0sg5ib/PcW82ACTndM+9GJ52+udiSMywAQCwAoENAIAFCGwAACxAYAMAYAEWnQEAkEGnW6xrhs5sMS8zbAAALEBgAwBgAQIbAAALcA0bSRn9AAAepAIgm431UJQz+bn46eN4cAoAAFMIgQ0AgAUIbAAALEBgAwBgARadAQCQolQX3n76OB6cAgDAFEJgAwBgAQIbAAALENgAAFiAwAYAwAIENgAAFiCwAQCwAIENAIAFCGwAACxAYAMAYAECGwAACyQV2KFQSNdcc40KCgo0e/Zs3X777Tpy5EhcH2OMGhoa5Pf7lZeXp8rKSnV3d6e1aAAAsk1Sgd3e3q61a9fqpZdeUltbm4aGhlRdXa2PPvrI6dPY2KimpiY1Nzers7NTPp9PVVVVGhgYSHvxOPtyYsNxGwDg7HAZY0yqB//xj3/U7Nmz1d7erhtuuEHGGPn9ftXW1mrjxo2SpFgsJq/Xqy1btmj16tWnPWc0GpXH49H1N9yv3NwZqZaG0xh256R0HCENAOk1NPSxDuzfrEgkosLCwlP2G9c17EgkIkmaNWuWJKmnp0fhcFjV1dVOH7fbrYqKCnV0dIx5jlgspmg0GrcBAIB4KQe2MUYbNmzQ4sWLVVpaKkkKh8OSJK/XG9fX6/U6+0YLhULyeDzOVlxcnGpJAABMWSkH9rp16/Tb3/5W//RP/5Swz+Vyxb02xiS0nVRfX69IJOJsvb29qZYEAMCUlZvKQd/4xjf0zDPPaP/+/brgggucdp/PJ+mTmXZRUZHT3tfXlzDrPsntdsvtdqdSBgAAWSOpGbYxRuvWrdPevXv1q1/9SiUlJXH7S0pK5PP51NbW5rQNDg6qvb1d5eXl6akYAIAslNQMe+3atWppadHPfvYzFRQUONelPR6P8vLy5HK5VFtbq2AwqEAgoEAgoGAwqPz8fC1fvjwjXwAAANkgqcDetm2bJKmysjKufefOnbr33nslSXV1dTpx4oRqamrU39+vsrIytba2qqCgIC0FAwCQjcZ1H3YmTOX7sI9dG//1+DtOxL0208ZemJcJZ3If9rT/Hkloc41Mqm8XALDeWbkPGwAAnB0ENgAAFiCwAQCwAIENAIAFUnpwClLjGrWG62wuMksFC8wAYPJghg0AgAUIbAAALEBgAwBgAa5hn0W+gx9P2HufyYNScmLDZ6ESAEAqmGEDAGABAhsAAAsQ2AAAWIBr2FPQWNerR3+QB/dYA4BdmGEDAGABAhsAAAsQ2AAAWIDABgDAAiw6m4J4AAoATD3MsAEAsACBDQCABQhsAAAsQGADAGABAhsAAAsQ2AAAWIDABgDAAgQ2AAAWILABALAAgQ0AgAUIbAAALEBgAwBgAQIbAAALENgAAFiAwAYAwAIENgAAFkgqsLdt26Yrr7xShYWFKiws1HXXXadnn33W2W+MUUNDg/x+v/Ly8lRZWanu7u60Fw0AQLZJKrAvuOACPfLII3r55Zf18ssv6y//8i912223OaHc2NiopqYmNTc3q7OzUz6fT1VVVRoYGMhI8QAAZAuXMcaM5wSzZs3S9773Pa1atUp+v1+1tbXauHGjJCkWi8nr9WrLli1avXr1mMfHYjHFYjHndTQaVXFxsa6/4X7l5s4YT2kAAEx6Q0Mf68D+zYpEIiosLDxlv5SvYQ8PD2vPnj366KOPdN1116mnp0fhcFjV1dVOH7fbrYqKCnV0dJzyPKFQSB6Px9mKi4tTLQkAgCkr6cA+fPiwPve5z8ntdmvNmjV66qmndNlllykcDkuSvF5vXH+v1+vsG0t9fb0ikYiz9fb2JlsSAABTXm6yB/z5n/+5urq69Kc//Uk//elPtXLlSrW3tzv7XS5XXH9jTELbp7ndbrnd7mTLAAAgqyQ9w54+fbq++MUvauHChQqFQvrSl76kf/iHf5DP55OkhNl0X19fwqwbAAAkZ9z3YRtjFIvFVFJSIp/Pp7a2Nmff4OCg2tvbVV5ePt63AQAgqyX1J/HvfOc7WrJkiYqLizUwMKA9e/bohRde0HPPPSeXy6Xa2loFg0EFAgEFAgEFg0Hl5+dr+fLlmaofAICskFRg/+EPf9A999yjY8eOyePx6Morr9Rzzz2nqqoqSVJdXZ1OnDihmpoa9ff3q6ysTK2trSooKMhI8QAAZItx34edbtFoVB6Ph/uwAQBZIeP3YQMAgLOHwAYAwAIENgAAFiCwAQCwAIENAIAFCGwAACxAYAMAYAECGwAACxDYAABYgMAGAMACBDYAABYgsAEAsEBSn9aF9Mr59Stn7b2Gb5x/1t4LAJB+zLABALAAgQ0AgAUIbAAALMA17LNo9DXr59/rysj7fNl/VUbOCwCYOMywAQCwAIENAIAFCGwAACxAYAMAYAEWnU1BYy1mu3kFD04BAJsxwwYAwAIENgAAFiCwAQCwAIENAIAFCGwAACxAYAMAYAECGwAACxDYAABYgMAGAMACBDYAABYgsAEAsMC4AjsUCsnlcqm2ttZpM8aooaFBfr9feXl5qqysVHd393jrBAAgq6X84R+dnZ3avn27rrzyyrj2xsZGNTU16fHHH9fFF1+shx56SFVVVTpy5IgKCgrGXXC2+7L/qtQOvDGtZQBTXs6vX4l7PfaH6qxK+rzTe/sT2n6x/6lxnxdTX0oz7A8//FArVqzQjh079PnPf95pN8Zo69at2rRpk5YtW6bS0lLt2rVLx48fV0tLS9qKBgAg26QU2GvXrtXSpUt18803x7X39PQoHA6rurraaXO73aqoqFBHR8eY54rFYopGo3EbAACIl/SfxPfs2aNXXnlFnZ2dCfvC4bAkyev1xrV7vV4dPXp0zPOFQiE9+OCDyZYBAEBWSWqG3dvbq/Xr12v37t2aMWPGKfu5XK6418aYhLaT6uvrFYlEnK23tzeZkgAAyApJzbAPHTqkvr4+LViwwGkbHh7W/v371dzcrCNHjkj6ZKZdVFTk9Onr60uYdZ/kdrvldrtTqT0jRi80kaThG+en5TzpMNbCl9EL0VKpF0C8sf6tpeKcD47HvT6ybuyfhcDpJDXDvummm3T48GF1dXU528KFC7VixQp1dXVp3rx58vl8amtrc44ZHBxUe3u7ysvL0148AADZIqkZdkFBgUpLS+PaZs6cqXPPPddpr62tVTAYVCAQUCAQUDAYVH5+vpYvX56+qgEAyDIp34d9KnV1dTpx4oRqamrU39+vsrIytba2cg82AADjMO7AfuGFF+Jeu1wuNTQ0qKGhYbynPivSda159MMQhsfok8o1sZQflAIg43pumx73+ou1LyX0eXbUv/uL/nnNac871s+lt7ZeG/e65GeDZ1AhphKeJQ4AgAUIbAAALEBgAwBgAQIbAAALpH2VuO3eX31dQtvsf038dJ3R/nCTL+71oVGfvpOues774W8S+vCgFGB8/nsmPwox+THDBgDAAgQ2AAAWILABALAAF25GOfTAtoS2M3l4yaHWrvQXo7GvWQNIr/bt2ye6BOC0mGEDAGABAhsAAAsQ2AAAWIDABgDAAiw6OwOpfMoWAIzlP7/y2ESXAEsxwwYAwAIENgAAFiCwAQCwANewLcMHfQCTx+jr0V+uvSqhz+gHL7EmBqlihg0AgAUIbAAALEBgAwBgAQIbAAALZP2is9GLuC791ysS+ry+6P+elVrG+lQwFpkBk9el/3pP3Osv3HjOaY+56J+vTWgbvXhtrIVp8//3dckVhymHGTYAABYgsAEAsACBDQCABbL+GvZoX3h0jGtQi85+HQAmvzF/XqTBWOtZZt0Yy8h7wR7MsAEAsACBDQCABQhsAAAswDXsM5Cph/ePPi/3XAMTY8ktyxPanv1FywRU8gl+FmAszLABALAAgQ0AgAUIbAAALJBUYDc0NMjlcsVtPp/P2W+MUUNDg/x+v/Ly8lRZWanu7u60Fw0AQLZJetHZ5Zdfrn379jmvc3JynP9ubGxUU1OTHn/8cV188cV66KGHVFVVpSNHjqigoCA9Ff9/+378f9J6vs8yenHYWA81GL0Qbaw+pz/m9H0ATB6p/By66J/XZOS8sFd0YESfv/j0/ZL+k3hubq58Pp+znX/++ZI+mV1v3bpVmzZt0rJly1RaWqpdu3bp+PHjammZuNWWAABMBUkH9ptvvim/36+SkhLdeeedevvttyVJPT09CofDqq6udvq63W5VVFSoo6PjlOeLxWKKRqNxGwAAiJdUYJeVlemJJ57Q888/rx07digcDqu8vFwffPCBwuGwJMnr9cYd4/V6nX1jCYVC8ng8zlZcXJzClwEAwNTmMsaYVA/+6KOPdNFFF6murk7XXnutFi1apPfee09FRUVOn69//evq7e3Vc889N+Y5YrGYYrH/eah9NBpVcXGxrr/hfuXmzki1tIzK+fUrCW1vPhH/oIN5OxOPc7/9x6TfKzbv/KSPwcT4zxXxv/8Wdk9P6HN+18dnqxyM05lcR755xaqkz9tzW+L3xX9+5bFxnxf2Ghr6WAf2b1YkElFhYeEp+43rtq6ZM2fqiiuu0JtvvumsFh89m+7r60uYdX+a2+1WYWFh3AYAAOKNK7BjsZhef/11FRUVqaSkRD6fT21tbc7+wcFBtbe3q7y8fNyFAgCQzZK6retv/uZvdOutt2rOnDnq6+vTQw89pGg0qpUrV8rlcqm2tlbBYFCBQECBQEDBYFD5+flavjzxOb0AAODMJRXY7777ru666y69//77Ov/883XttdfqpZde0ty5cyVJdXV1OnHihGpqatTf36+ysjK1tram/R5sAACyTVKBvWfPns/c73K51NDQoIaGhvHUZKW3b45foHLzzsRFIywgm9ou+vHIqBYWmH2W0Yu6WGiFMzHWYsBs+d7hWeIAAFiAwAYAwAIENgAAFkj6wz8wtoQP+7hxQsrAGXhv7WDca/8PEh9kgcw7k+uOE/n/quTp/3XaPhdpKOnzznrVlfhe0+PfK5XzZotsuV49FmbYAABYgMAGAMACBDYAABYgsAEAsACLzlIwfOP803fCpMUiM3tM5P+ri36SmYVfnt/FxmjLyFthimGGDQCABQhsAAAsQGADAGABAhsAAAsQ2AAAWIDABgDAAgQ2AAAWILABALAAgQ0AgAUIbAAALEBgAwBgAQIbAAALENgAAFiAwAYAwAIENgAAFiCwAQCwAIENAIAFCGwAACxAYAMAYAECGwAACxDYAABYgMAGAMACBDYAABYgsAEAsEDSgf373/9ed999t84991zl5+frqquu0qFDh5z9xhg1NDTI7/crLy9PlZWV6u7uTmvRAABkm6QCu7+/X4sWLdI555yjZ599Vq+99pr+/u//Xn/2Z3/m9GlsbFRTU5Oam5vV2dkpn8+nqqoqDQwMpLt2AACyRm4ynbds2aLi4mLt3LnTabvwwgud/zbGaOvWrdq0aZOWLVsmSdq1a5e8Xq9aWlq0evXq9FQNAECWSWqG/cwzz2jhwoW64447NHv2bF199dXasWOHs7+np0fhcFjV1dVOm9vtVkVFhTo6OsY8ZywWUzQajdsAAEC8pAL77bff1rZt2xQIBPT8889rzZo1+uY3v6knnnhCkhQOhyVJXq837jiv1+vsGy0UCsnj8ThbcXFxKl8HAABTWlKBPTIyovnz5ysYDOrqq6/W6tWr9fWvf13btm2L6+dyueJeG2MS2k6qr69XJBJxtt7e3iS/BAAApr6kAruoqEiXXXZZXNull16qd955R5Lk8/kkKWE23dfXlzDrPsntdquwsDBuAwAA8ZIK7EWLFunIkSNxbW+88Ybmzp0rSSopKZHP51NbW5uzf3BwUO3t7SovL09DuQAAZKekVol/61vfUnl5uYLBoL7yla/o4MGD2r59u7Zv3y7pkz+F19bWKhgMKhAIKBAIKBgMKj8/X8uXL8/IFwAAQDZIKrCvueYaPfXUU6qvr9fmzZtVUlKirVu3asWKFU6furo6nThxQjU1Nerv71dZWZlaW1tVUFCQ9uIBAMgWLmOMmegiPi0ajcrj8ej6G+5Xbu6MiS4HAICMGhr6WAf2b1YkEvnMdVw8SxwAAAsQ2AAAWIDABgDAAgQ2AAAWILABALAAgQ0AgAUIbAAALEBgAwBgAQIbAAALENgAAFiAwAYAwAIENgAAFiCwAQCwAIENAIAFCGwAACxAYAMAYAECGwAACxDYAABYgMAGAMACBDYAABYgsAEAsACBDQCABQhsAAAsQGADAGABAhsAAAsQ2AAAWIDABgDAAgQ2AAAWILABALAAgQ0AgAUIbAAALEBgAwBgAQIbAAALENgAAFggqcC+8MIL5XK5Era1a9dKkowxamhokN/vV15eniorK9Xd3Z2RwgEAyCZJBXZnZ6eOHTvmbG1tbZKkO+64Q5LU2NiopqYmNTc3q7OzUz6fT1VVVRoYGEh/5QAAZJGkAvv888+Xz+dztp///Oe66KKLVFFRIWOMtm7dqk2bNmnZsmUqLS3Vrl27dPz4cbW0tGSqfgAAskLK17AHBwe1e/durVq1Si6XSz09PQqHw6qurnb6uN1uVVRUqKOj45TnicViikajcRsAAIiXcmA//fTT+tOf/qR7771XkhQOhyVJXq83rp/X63X2jSUUCsnj8ThbcXFxqiUBADBlpRzYP/rRj7RkyRL5/f64dpfLFffaGJPQ9mn19fWKRCLO1tvbm2pJAABMWbmpHHT06FHt27dPe/fuddp8Pp+kT2baRUVFTntfX1/CrPvT3G633G53KmUAAJA1Upph79y5U7Nnz9bSpUudtpKSEvl8PmfluPTJde729naVl5ePv1IAALJY0jPskZER7dy5UytXrlRu7v8c7nK5VFtbq2AwqEAgoEAgoGAwqPz8fC1fvjytRQMAkG2SDux9+/bpnXfe0apVqxL21dXV6cSJE6qpqVF/f7/KysrU2tqqgoKCtBQLAEC2chljzEQX8WnRaFQej0fX33C/cnNnTHQ5AABk1NDQxzqwf7MikYgKCwtP2Y9niQMAYAECGwAACxDYAABYgMAGAMACBDYAABYgsAEAsACBDQCABQhsAAAsQGADAGABAhsAAAsQ2AAAWIDABgDAAgQ2AAAWILABALAAgQ0AgAUIbAAALEBgAwBgAQIbAAALENgAAFiAwAYAwAIENgAAFiCwAQCwAIENAIAFCGwAACxAYAMAYAECGwAACxDYAABYgMAGAMACBDYAABYgsAEAsACBDQCABQhsAAAsQGADAGCBpAJ7aGhI3/3ud1VSUqK8vDzNmzdPmzdv1sjIiNPHGKOGhgb5/X7l5eWpsrJS3d3daS8cAIBsklRgb9myRY899piam5v1+uuvq7GxUd/73vf0/e9/3+nT2NiopqYmNTc3q7OzUz6fT1VVVRoYGEh78QAAZIukAvs3v/mNbrvtNi1dulQXXnih/uqv/krV1dV6+eWXJX0yu966das2bdqkZcuWqbS0VLt27dLx48fV0tKSkS8AAIBskFRgL168WL/85S/1xhtvSJJeffVVvfjii7rlllskST09PQqHw6qurnaOcbvdqqioUEdHx5jnjMViikajcRsAAIiXm0znjRs3KhKJ6JJLLlFOTo6Gh4f18MMP66677pIkhcNhSZLX6407zuv16ujRo2OeMxQK6cEHH0yldgAAskZSM+wnn3xSu3fvVktLi1555RXt2rVLf/d3f6ddu3bF9XO5XHGvjTEJbSfV19crEok4W29vb5JfAgAAU19SM+xvf/vbuu+++3TnnXdKkq644godPXpUoVBIK1eulM/nk/TJTLuoqMg5rq+vL2HWfZLb7Zbb7U61fgAAskJSM+zjx49r2rT4Q3JycpzbukpKSuTz+dTW1ubsHxwcVHt7u8rLy9NQLgAA2SmpGfatt96qhx9+WHPmzNHll1+uf//3f1dTU5NWrVol6ZM/hdfW1ioYDCoQCCgQCCgYDCo/P1/Lly/PyBcAAEA2SCqwv//97+tv//ZvVVNTo76+Pvn9fq1evVr333+/06eurk4nTpxQTU2N+vv7VVZWptbWVhUUFKS9eAAAsoXLGGMmuohPi0aj8ng8uv6G+5WbO2OiywEAIKOGhj7Wgf2bFYlEVFhYeMp+PEscAAALENgAAFiAwAYAwAJJLTo7G05eUh8aik1wJQAAZN7JvDvdkrJJt+js3XffVXFx8USXAQDAWdXb26sLLrjglPsnXWCPjIzovffeU0FBgQYGBlRcXKze3t7PXDmH1EWjUcY4wxjjzGOMM48xzhxjjAYGBuT3+xMeTvZpk+5P4tOmTXN+wzj5/PHCwkK+QTKMMc48xjjzGOPMY4wzw+PxnLYPi84AALAAgQ0AgAUmdWC73W498MADfJpXBjHGmccYZx5jnHmM8cSbdIvOAABAokk9wwYAAJ8gsAEAsACBDQCABQhsAAAsQGADAGCBSRvYjz76qEpKSjRjxgwtWLBABw4cmOiSrBUKhXTNNdeooKBAs2fP1u23364jR47E9THGqKGhQX6/X3l5eaqsrFR3d/cEVWy/UCgkl8ul2tpap40xHr/f//73uvvuu3XuuecqPz9fV111lQ4dOuTsZ4zHZ2hoSN/97ndVUlKivLw8zZs3T5s3b9bIyIjThzGeQGYS2rNnjznnnHPMjh07zGuvvWbWr19vZs6caY4ePTrRpVnpy1/+stm5c6f5j//4D9PV1WWWLl1q5syZYz788EOnzyOPPGIKCgrMT3/6U3P48GHz1a9+1RQVFZloNDqBldvp4MGD5sILLzRXXnmlWb9+vdPOGI/Pf/3Xf5m5c+eae++91/zbv/2b6enpMfv27TNvvfWW04cxHp+HHnrInHvuuebnP/+56enpMT/5yU/M5z73ObN161anD2M8cSZlYP/FX/yFWbNmTVzbJZdcYu67774Jqmhq6evrM5JMe3u7McaYkZER4/P5zCOPPOL0+fjjj43H4zGPPfbYRJVppYGBARMIBExbW5upqKhwApsxHr+NGzeaxYsXn3I/Yzx+S5cuNatWrYprW7Zsmbn77ruNMYzxRJt0fxIfHBzUoUOHVF1dHddeXV2tjo6OCapqaolEIpKkWbNmSZJ6enoUDofjxtztdquiooIxT9LatWu1dOlS3XzzzXHtjPH4PfPMM1q4cKHuuOMOzZ49W1dffbV27Njh7GeMx2/x4sX65S9/qTfeeEOS9Oqrr+rFF1/ULbfcIokxnmiT7tO63n//fQ0PD8vr9ca1e71ehcPhCapq6jDGaMOGDVq8eLFKS0slyRnXscb86NGjZ71GW+3Zs0evvPKKOjs7E/YxxuP39ttva9u2bdqwYYO+853v6ODBg/rmN78pt9utr33ta4xxGmzcuFGRSESXXHKJcnJyNDw8rIcfflh33XWXJL6PJ9qkC+yTTn605knGmIQ2JG/dunX67W9/qxdffDFhH2Oeut7eXq1fv16tra2aMWPGKfsxxqkbGRnRwoULFQwGJUlXX321uru7tW3bNn3ta19z+jHGqXvyySe1e/dutbS06PLLL1dXV5dqa2vl9/u1cuVKpx9jPDEm3Z/EzzvvPOXk5CTMpvv6+hJ+q0NyvvGNb+iZZ57Rr3/9a+czxyXJ5/NJEmM+DocOHVJfX58WLFig3Nxc5ebmqr29Xf/4j/+o3NxcZxwZ49QVFRXpsssui2u79NJL9c4770ji+zgdvv3tb+u+++7TnXfeqSuuuEL33HOPvvWtbykUCklijCfapAvs6dOna8GCBWpra4trb2trU3l5+QRVZTdjjNatW6e9e/fqV7/6lUpKSuL2l5SUyOfzxY354OCg2tvbGfMzdNNNN+nw4cPq6upytoULF2rFihXq6urSvHnzGONxWrRoUcLtiG+88Ybmzp0rie/jdDh+/LimTYuPhZycHOe2LsZ4gk3ggrdTOnlb149+9CPz2muvmdraWjNz5kzzu9/9bqJLs9Jf//VfG4/HY1544QVz7NgxZzt+/LjT55FHHjEej8fs3bvXHD582Nx1113cqjFOn14lbgxjPF4HDx40ubm55uGHHzZvvvmm+fGPf2zy8/PN7t27nT6M8fisXLnSfOELX3Bu69q7d68577zzTF1dndOHMZ44kzKwjTHmBz/4gZk7d66ZPn26mT9/vnMLEpInacxt586dTp+RkRHzwAMPGJ/PZ9xut7nhhhvM4cOHJ67oKWB0YDPG4/cv//IvprS01LjdbnPJJZeY7du3x+1njMcnGo2a9evXmzlz5pgZM2aYefPmmU2bNplYLOb0YYwnDp+HDQCABSbdNWwAAJCIwAYAwAIENgAAFiCwAQCwAIENAIAFCGwAACxAYAMAYAECGwAACxDYAABYgMAGAMACBDYAABb4f1GK+Bc3DVsZAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(obs[0]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "2390865e-c165-4893-87c4-6b37cacd3eb6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "done, done_cap = env.get_done()\n",
    "print(done)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "cb21821e-2efb-4845-9528-473e78bbe3fb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total reward for episode 1 is -88\n",
      "Total reward for episode 2 is -85\n",
      "Total reward for episode 3 is -71\n"
     ]
    }
   ],
   "source": [
    "for episode in range(3):\n",
    "    obs, info = env.reset()\n",
    "    done = False\n",
    "    total_reward = 0\n",
    "    while not done:\n",
    "        obs, reward, done, _, info = env.step(env.action_space.sample())\n",
    "        total_reward += reward\n",
    "    print(f'Total reward for episode {episode + 1 } is {total_reward}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "c520653d-138e-46ff-a2aa-6bf55a7f4733",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/ahmet/.local/lib/python3.10/site-packages/tensorflow/python/framework/dtypes.py:35: DeprecationWarning: ml_dtypes.float8_e4m3b11 is deprecated. Use ml_dtypes.float8_e4m3b11fnuz\n",
      "  from tensorflow.tsl.python.lib.core import pywrap_ml_dtypes\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from stable_baselines3.common.callbacks import BaseCallback\n",
    "from stable_baselines3.common import env_checker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "7c28231b-9a45-4d2d-a5f2-a077ce3c3991",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Checking environment\n",
    "env_checker.check_env(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "e76011d7-224b-46dc-9ea8-a92ab4c6c8f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the model Callbacks to the directories that we will define\n",
    "class TrainAndLoggingCallback(BaseCallback):\n",
    "\n",
    "    def __init__(self, check_freq, save_path, verbose=1):\n",
    "        super(TrainAndLoggingCallback, self).__init__(verbose)\n",
    "        self.check_freq = check_freq\n",
    "        self.save_path = save_path\n",
    "\n",
    "    def _init_callback(self):\n",
    "        if self.save_path is not None:\n",
    "            os.makedirs(self.save_path, exist_ok=True)\n",
    "\n",
    "    def _on_step(self):\n",
    "        if self.n_calls % self.check_freq == 0:\n",
    "            model_path = os.path.join(self.save_path, 'best_model_{}'.format(self.n_calls))\n",
    "            self.model.save(model_path)\n",
    "\n",
    "        return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "2d8b7a75-bffc-4545-a266-8bc95cac3397",
   "metadata": {},
   "outputs": [],
   "source": [
    "checkpoint_dir = './train/'\n",
    "log_dir = './logs/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2cdd9eb7-4508-49c6-8f68-fd9b07422d0a",
   "metadata": {},
   "outputs": [],
   "source": [
    "callback = TrainAndLoggingCallback(check_freq = 1000, save_path = checkpoint_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "c7139d3e-3a92-4ce4-89b3-d0dd44a736bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from stable_baselines3 import DQN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d7293195-a66b-41c7-b4cb-ee2c08ec3668",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cpu device\n",
      "Wrapping the env with a `Monitor` wrapper\n",
      "Wrapping the env in a DummyVecEnv.\n"
     ]
    }
   ],
   "source": [
    "# define DQN model\n",
    "model = DQN('CnnPolicy', env, tensorboard_log = log_dir, verbose = 1,\n",
    "            buffer_size=150000, learning_starts = 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5cd32467-df50-4b1c-b04b-7fbe137e9f1a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logging to ./logs/DQN_11\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.5     |\n",
      "|    ep_rew_mean      | -75      |\n",
      "|    exploration_rate | 0.996    |\n",
      "| time/               |          |\n",
      "|    episodes         | 4        |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 33       |\n",
      "|    total_timesteps  | 70       |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.1     |\n",
      "|    ep_rew_mean      | -73.9    |\n",
      "|    exploration_rate | 0.991    |\n",
      "| time/               |          |\n",
      "|    episodes         | 8        |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 60       |\n",
      "|    total_timesteps  | 137      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.2     |\n",
      "|    ep_rew_mean      | -75.4    |\n",
      "|    exploration_rate | 0.988    |\n",
      "| time/               |          |\n",
      "|    episodes         | 12       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 87       |\n",
      "|    total_timesteps  | 195      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.2     |\n",
      "|    ep_rew_mean      | -74.8    |\n",
      "|    exploration_rate | 0.984    |\n",
      "| time/               |          |\n",
      "|    episodes         | 16       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 113      |\n",
      "|    total_timesteps  | 259      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.6     |\n",
      "|    ep_rew_mean      | -73.4    |\n",
      "|    exploration_rate | 0.979    |\n",
      "| time/               |          |\n",
      "|    episodes         | 20       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 141      |\n",
      "|    total_timesteps  | 332      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.4     |\n",
      "|    ep_rew_mean      | -73.9    |\n",
      "|    exploration_rate | 0.975    |\n",
      "| time/               |          |\n",
      "|    episodes         | 24       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 168      |\n",
      "|    total_timesteps  | 393      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16       |\n",
      "|    ep_rew_mean      | -74.8    |\n",
      "|    exploration_rate | 0.972    |\n",
      "| time/               |          |\n",
      "|    episodes         | 28       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 193      |\n",
      "|    total_timesteps  | 448      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.1     |\n",
      "|    ep_rew_mean      | -74.3    |\n",
      "|    exploration_rate | 0.967    |\n",
      "| time/               |          |\n",
      "|    episodes         | 32       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 221      |\n",
      "|    total_timesteps  | 516      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.1     |\n",
      "|    ep_rew_mean      | -74.3    |\n",
      "|    exploration_rate | 0.963    |\n",
      "| time/               |          |\n",
      "|    episodes         | 36       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 247      |\n",
      "|    total_timesteps  | 578      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 15.9     |\n",
      "|    ep_rew_mean      | -74.8    |\n",
      "|    exploration_rate | 0.96     |\n",
      "| time/               |          |\n",
      "|    episodes         | 40       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 273      |\n",
      "|    total_timesteps  | 635      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 15.8     |\n",
      "|    ep_rew_mean      | -74.7    |\n",
      "|    exploration_rate | 0.956    |\n",
      "| time/               |          |\n",
      "|    episodes         | 44       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 299      |\n",
      "|    total_timesteps  | 697      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.1     |\n",
      "|    ep_rew_mean      | -73.9    |\n",
      "|    exploration_rate | 0.951    |\n",
      "| time/               |          |\n",
      "|    episodes         | 48       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 327      |\n",
      "|    total_timesteps  | 773      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.2     |\n",
      "|    ep_rew_mean      | -73.6    |\n",
      "|    exploration_rate | 0.947    |\n",
      "| time/               |          |\n",
      "|    episodes         | 52       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 356      |\n",
      "|    total_timesteps  | 844      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.1     |\n",
      "|    ep_rew_mean      | -73.9    |\n",
      "|    exploration_rate | 0.943    |\n",
      "| time/               |          |\n",
      "|    episodes         | 56       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 382      |\n",
      "|    total_timesteps  | 904      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.1     |\n",
      "|    ep_rew_mean      | -73.9    |\n",
      "|    exploration_rate | 0.939    |\n",
      "| time/               |          |\n",
      "|    episodes         | 60       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 407      |\n",
      "|    total_timesteps  | 966      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16       |\n",
      "|    ep_rew_mean      | -74.1    |\n",
      "|    exploration_rate | 0.935    |\n",
      "| time/               |          |\n",
      "|    episodes         | 64       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 433      |\n",
      "|    total_timesteps  | 1023     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 7.45     |\n",
      "|    n_updates        | 5        |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16       |\n",
      "|    ep_rew_mean      | -73.9    |\n",
      "|    exploration_rate | 0.931    |\n",
      "| time/               |          |\n",
      "|    episodes         | 68       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 458      |\n",
      "|    total_timesteps  | 1087     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 13       |\n",
      "|    n_updates        | 21       |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 15.9     |\n",
      "|    ep_rew_mean      | -74.1    |\n",
      "|    exploration_rate | 0.927    |\n",
      "| time/               |          |\n",
      "|    episodes         | 72       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 485      |\n",
      "|    total_timesteps  | 1145     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0177   |\n",
      "|    n_updates        | 36       |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 15.9     |\n",
      "|    ep_rew_mean      | -74      |\n",
      "|    exploration_rate | 0.923    |\n",
      "| time/               |          |\n",
      "|    episodes         | 76       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 513      |\n",
      "|    total_timesteps  | 1212     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 12.6     |\n",
      "|    n_updates        | 52       |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16       |\n",
      "|    ep_rew_mean      | -73.7    |\n",
      "|    exploration_rate | 0.919    |\n",
      "| time/               |          |\n",
      "|    episodes         | 80       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 540      |\n",
      "|    total_timesteps  | 1281     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.14     |\n",
      "|    n_updates        | 70       |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 15.9     |\n",
      "|    ep_rew_mean      | -74      |\n",
      "|    exploration_rate | 0.915    |\n",
      "| time/               |          |\n",
      "|    episodes         | 84       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 568      |\n",
      "|    total_timesteps  | 1338     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 6.42     |\n",
      "|    n_updates        | 84       |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 15.9     |\n",
      "|    ep_rew_mean      | -73.9    |\n",
      "|    exploration_rate | 0.911    |\n",
      "| time/               |          |\n",
      "|    episodes         | 88       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 593      |\n",
      "|    total_timesteps  | 1402     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 9.41     |\n",
      "|    n_updates        | 100      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16       |\n",
      "|    ep_rew_mean      | -73.5    |\n",
      "|    exploration_rate | 0.907    |\n",
      "| time/               |          |\n",
      "|    episodes         | 92       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 621      |\n",
      "|    total_timesteps  | 1475     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 9.42     |\n",
      "|    n_updates        | 118      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.1     |\n",
      "|    ep_rew_mean      | -73.4    |\n",
      "|    exploration_rate | 0.902    |\n",
      "| time/               |          |\n",
      "|    episodes         | 96       |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 651      |\n",
      "|    total_timesteps  | 1546     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.14     |\n",
      "|    n_updates        | 136      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.1     |\n",
      "|    ep_rew_mean      | -73.5    |\n",
      "|    exploration_rate | 0.898    |\n",
      "| time/               |          |\n",
      "|    episodes         | 100      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 676      |\n",
      "|    total_timesteps  | 1605     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 6.28     |\n",
      "|    n_updates        | 151      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 15.9     |\n",
      "|    ep_rew_mean      | -73.5    |\n",
      "|    exploration_rate | 0.895    |\n",
      "| time/               |          |\n",
      "|    episodes         | 104      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 702      |\n",
      "|    total_timesteps  | 1664     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.14     |\n",
      "|    n_updates        | 165      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16       |\n",
      "|    ep_rew_mean      | -73.4    |\n",
      "|    exploration_rate | 0.89     |\n",
      "| time/               |          |\n",
      "|    episodes         | 108      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 730      |\n",
      "|    total_timesteps  | 1733     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00525  |\n",
      "|    n_updates        | 183      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16       |\n",
      "|    ep_rew_mean      | -73.2    |\n",
      "|    exploration_rate | 0.886    |\n",
      "| time/               |          |\n",
      "|    episodes         | 112      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 756      |\n",
      "|    total_timesteps  | 1794     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 12.6     |\n",
      "|    n_updates        | 198      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 15.9     |\n",
      "|    ep_rew_mean      | -73.3    |\n",
      "|    exploration_rate | 0.883    |\n",
      "| time/               |          |\n",
      "|    episodes         | 116      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 782      |\n",
      "|    total_timesteps  | 1854     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.00969  |\n",
      "|    n_updates        | 213      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 15.8     |\n",
      "|    ep_rew_mean      | -73.5    |\n",
      "|    exploration_rate | 0.879    |\n",
      "| time/               |          |\n",
      "|    episodes         | 120      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 808      |\n",
      "|    total_timesteps  | 1917     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 12.6     |\n",
      "|    n_updates        | 229      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 15.9     |\n",
      "|    ep_rew_mean      | -73.1    |\n",
      "|    exploration_rate | 0.874    |\n",
      "| time/               |          |\n",
      "|    episodes         | 124      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 834      |\n",
      "|    total_timesteps  | 1986     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 12.5     |\n",
      "|    n_updates        | 246      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.1     |\n",
      "|    ep_rew_mean      | -72.6    |\n",
      "|    exploration_rate | 0.87     |\n",
      "| time/               |          |\n",
      "|    episodes         | 128      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 862      |\n",
      "|    total_timesteps  | 2059     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 9.44     |\n",
      "|    n_updates        | 264      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.1     |\n",
      "|    ep_rew_mean      | -72.4    |\n",
      "|    exploration_rate | 0.865    |\n",
      "| time/               |          |\n",
      "|    episodes         | 132      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 888      |\n",
      "|    total_timesteps  | 2131     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.13     |\n",
      "|    n_updates        | 282      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.1     |\n",
      "|    ep_rew_mean      | -72.4    |\n",
      "|    exploration_rate | 0.861    |\n",
      "| time/               |          |\n",
      "|    episodes         | 136      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 915      |\n",
      "|    total_timesteps  | 2192     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 12.6     |\n",
      "|    n_updates        | 297      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.3     |\n",
      "|    ep_rew_mean      | -71.8    |\n",
      "|    exploration_rate | 0.856    |\n",
      "| time/               |          |\n",
      "|    episodes         | 140      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 943      |\n",
      "|    total_timesteps  | 2269     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 6.19     |\n",
      "|    n_updates        | 317      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.4     |\n",
      "|    ep_rew_mean      | -71.4    |\n",
      "|    exploration_rate | 0.852    |\n",
      "| time/               |          |\n",
      "|    episodes         | 144      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 969      |\n",
      "|    total_timesteps  | 2342     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 6.15     |\n",
      "|    n_updates        | 335      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.4     |\n",
      "|    ep_rew_mean      | -71.4    |\n",
      "|    exploration_rate | 0.847    |\n",
      "| time/               |          |\n",
      "|    episodes         | 148      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 996      |\n",
      "|    total_timesteps  | 2414     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.13     |\n",
      "|    n_updates        | 353      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.4     |\n",
      "|    ep_rew_mean      | -71.3    |\n",
      "|    exploration_rate | 0.843    |\n",
      "| time/               |          |\n",
      "|    episodes         | 152      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1024     |\n",
      "|    total_timesteps  | 2486     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.12     |\n",
      "|    n_updates        | 371      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.5     |\n",
      "|    ep_rew_mean      | -71      |\n",
      "|    exploration_rate | 0.838    |\n",
      "| time/               |          |\n",
      "|    episodes         | 156      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1051     |\n",
      "|    total_timesteps  | 2554     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 6.32     |\n",
      "|    n_updates        | 388      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.6     |\n",
      "|    ep_rew_mean      | -70.7    |\n",
      "|    exploration_rate | 0.834    |\n",
      "| time/               |          |\n",
      "|    episodes         | 160      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1076     |\n",
      "|    total_timesteps  | 2623     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.11     |\n",
      "|    n_updates        | 405      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.7     |\n",
      "|    ep_rew_mean      | -70.2    |\n",
      "|    exploration_rate | 0.829    |\n",
      "| time/               |          |\n",
      "|    episodes         | 164      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1102     |\n",
      "|    total_timesteps  | 2693     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3        |\n",
      "|    n_updates        | 423      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.7     |\n",
      "|    ep_rew_mean      | -70.1    |\n",
      "|    exploration_rate | 0.825    |\n",
      "| time/               |          |\n",
      "|    episodes         | 168      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1128     |\n",
      "|    total_timesteps  | 2761     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 14.7     |\n",
      "|    n_updates        | 440      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.8     |\n",
      "|    ep_rew_mean      | -69.9    |\n",
      "|    exploration_rate | 0.821    |\n",
      "| time/               |          |\n",
      "|    episodes         | 172      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1154     |\n",
      "|    total_timesteps  | 2823     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0855   |\n",
      "|    n_updates        | 455      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.7     |\n",
      "|    ep_rew_mean      | -70      |\n",
      "|    exploration_rate | 0.817    |\n",
      "| time/               |          |\n",
      "|    episodes         | 176      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1180     |\n",
      "|    total_timesteps  | 2885     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 5.42     |\n",
      "|    n_updates        | 471      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.6     |\n",
      "|    ep_rew_mean      | -70.2    |\n",
      "|    exploration_rate | 0.813    |\n",
      "| time/               |          |\n",
      "|    episodes         | 180      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1206     |\n",
      "|    total_timesteps  | 2945     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.98     |\n",
      "|    n_updates        | 486      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.9     |\n",
      "|    ep_rew_mean      | -69.6    |\n",
      "|    exploration_rate | 0.809    |\n",
      "| time/               |          |\n",
      "|    episodes         | 184      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1234     |\n",
      "|    total_timesteps  | 3023     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 9.48     |\n",
      "|    n_updates        | 505      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 16.9     |\n",
      "|    ep_rew_mean      | -69.3    |\n",
      "|    exploration_rate | 0.804    |\n",
      "| time/               |          |\n",
      "|    episodes         | 188      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1261     |\n",
      "|    total_timesteps  | 3097     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.02     |\n",
      "|    n_updates        | 524      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.1     |\n",
      "|    ep_rew_mean      | -69.1    |\n",
      "|    exploration_rate | 0.798    |\n",
      "| time/               |          |\n",
      "|    episodes         | 192      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1292     |\n",
      "|    total_timesteps  | 3183     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 8.19     |\n",
      "|    n_updates        | 545      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.1     |\n",
      "|    ep_rew_mean      | -69      |\n",
      "|    exploration_rate | 0.794    |\n",
      "| time/               |          |\n",
      "|    episodes         | 196      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1320     |\n",
      "|    total_timesteps  | 3253     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.38     |\n",
      "|    n_updates        | 563      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.1     |\n",
      "|    ep_rew_mean      | -68.8    |\n",
      "|    exploration_rate | 0.79     |\n",
      "| time/               |          |\n",
      "|    episodes         | 200      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1345     |\n",
      "|    total_timesteps  | 3316     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 5.37     |\n",
      "|    n_updates        | 578      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.2     |\n",
      "|    ep_rew_mean      | -68.4    |\n",
      "|    exploration_rate | 0.785    |\n",
      "| time/               |          |\n",
      "|    episodes         | 204      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1372     |\n",
      "|    total_timesteps  | 3388     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.48     |\n",
      "|    n_updates        | 596      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.2     |\n",
      "|    ep_rew_mean      | -68.3    |\n",
      "|    exploration_rate | 0.781    |\n",
      "| time/               |          |\n",
      "|    episodes         | 208      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1398     |\n",
      "|    total_timesteps  | 3457     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.208    |\n",
      "|    n_updates        | 614      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.3     |\n",
      "|    ep_rew_mean      | -68.1    |\n",
      "|    exploration_rate | 0.777    |\n",
      "| time/               |          |\n",
      "|    episodes         | 212      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1424     |\n",
      "|    total_timesteps  | 3526     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 6.39     |\n",
      "|    n_updates        | 631      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.4     |\n",
      "|    ep_rew_mean      | -67.8    |\n",
      "|    exploration_rate | 0.772    |\n",
      "| time/               |          |\n",
      "|    episodes         | 216      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1449     |\n",
      "|    total_timesteps  | 3593     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.69     |\n",
      "|    n_updates        | 648      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.6     |\n",
      "|    ep_rew_mean      | -67.5    |\n",
      "|    exploration_rate | 0.767    |\n",
      "| time/               |          |\n",
      "|    episodes         | 220      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1480     |\n",
      "|    total_timesteps  | 3672     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 8.55     |\n",
      "|    n_updates        | 667      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.5     |\n",
      "|    ep_rew_mean      | -67.6    |\n",
      "|    exploration_rate | 0.763    |\n",
      "| time/               |          |\n",
      "|    episodes         | 224      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1505     |\n",
      "|    total_timesteps  | 3737     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 5.62     |\n",
      "|    n_updates        | 684      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.4     |\n",
      "|    ep_rew_mean      | -67.8    |\n",
      "|    exploration_rate | 0.759    |\n",
      "| time/               |          |\n",
      "|    episodes         | 228      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1531     |\n",
      "|    total_timesteps  | 3801     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 6.01     |\n",
      "|    n_updates        | 700      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.4     |\n",
      "|    ep_rew_mean      | -68.1    |\n",
      "|    exploration_rate | 0.755    |\n",
      "| time/               |          |\n",
      "|    episodes         | 232      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1561     |\n",
      "|    total_timesteps  | 3871     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.02     |\n",
      "|    n_updates        | 717      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.4     |\n",
      "|    ep_rew_mean      | -68      |\n",
      "|    exploration_rate | 0.751    |\n",
      "| time/               |          |\n",
      "|    episodes         | 236      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1587     |\n",
      "|    total_timesteps  | 3933     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.82     |\n",
      "|    n_updates        | 733      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.4     |\n",
      "|    ep_rew_mean      | -68.1    |\n",
      "|    exploration_rate | 0.746    |\n",
      "| time/               |          |\n",
      "|    episodes         | 240      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1614     |\n",
      "|    total_timesteps  | 4005     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.54     |\n",
      "|    n_updates        | 751      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.3     |\n",
      "|    ep_rew_mean      | -68.4    |\n",
      "|    exploration_rate | 0.742    |\n",
      "| time/               |          |\n",
      "|    episodes         | 244      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1640     |\n",
      "|    total_timesteps  | 4069     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 5.76     |\n",
      "|    n_updates        | 767      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.3     |\n",
      "|    ep_rew_mean      | -68.3    |\n",
      "|    exploration_rate | 0.738    |\n",
      "| time/               |          |\n",
      "|    episodes         | 248      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1665     |\n",
      "|    total_timesteps  | 4141     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.463    |\n",
      "|    n_updates        | 785      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.4     |\n",
      "|    ep_rew_mean      | -68      |\n",
      "|    exploration_rate | 0.732    |\n",
      "| time/               |          |\n",
      "|    episodes         | 252      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1692     |\n",
      "|    total_timesteps  | 4224     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.1      |\n",
      "|    n_updates        | 805      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.4     |\n",
      "|    ep_rew_mean      | -67.7    |\n",
      "|    exploration_rate | 0.728    |\n",
      "| time/               |          |\n",
      "|    episodes         | 256      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1719     |\n",
      "|    total_timesteps  | 4298     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.48     |\n",
      "|    n_updates        | 824      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.4     |\n",
      "|    ep_rew_mean      | -67.7    |\n",
      "|    exploration_rate | 0.723    |\n",
      "| time/               |          |\n",
      "|    episodes         | 260      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1745     |\n",
      "|    total_timesteps  | 4368     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.98     |\n",
      "|    n_updates        | 841      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.5     |\n",
      "|    ep_rew_mean      | -67.6    |\n",
      "|    exploration_rate | 0.718    |\n",
      "| time/               |          |\n",
      "|    episodes         | 264      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1773     |\n",
      "|    total_timesteps  | 4446     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.429    |\n",
      "|    n_updates        | 861      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.6     |\n",
      "|    ep_rew_mean      | -67.4    |\n",
      "|    exploration_rate | 0.714    |\n",
      "| time/               |          |\n",
      "|    episodes         | 268      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1800     |\n",
      "|    total_timesteps  | 4520     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.943    |\n",
      "|    n_updates        | 879      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.7     |\n",
      "|    ep_rew_mean      | -67      |\n",
      "|    exploration_rate | 0.709    |\n",
      "| time/               |          |\n",
      "|    episodes         | 272      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1826     |\n",
      "|    total_timesteps  | 4596     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 5.15     |\n",
      "|    n_updates        | 898      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.9     |\n",
      "|    ep_rew_mean      | -66.5    |\n",
      "|    exploration_rate | 0.704    |\n",
      "| time/               |          |\n",
      "|    episodes         | 276      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1853     |\n",
      "|    total_timesteps  | 4672     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.457    |\n",
      "|    n_updates        | 917      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.9     |\n",
      "|    ep_rew_mean      | -66.5    |\n",
      "|    exploration_rate | 0.7      |\n",
      "| time/               |          |\n",
      "|    episodes         | 280      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1880     |\n",
      "|    total_timesteps  | 4733     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 6.26     |\n",
      "|    n_updates        | 933      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 17.9     |\n",
      "|    ep_rew_mean      | -66.4    |\n",
      "|    exploration_rate | 0.695    |\n",
      "| time/               |          |\n",
      "|    episodes         | 284      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1909     |\n",
      "|    total_timesteps  | 4817     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.883    |\n",
      "|    n_updates        | 954      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 18       |\n",
      "|    ep_rew_mean      | -66.2    |\n",
      "|    exploration_rate | 0.69     |\n",
      "| time/               |          |\n",
      "|    episodes         | 288      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1938     |\n",
      "|    total_timesteps  | 4897     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.7      |\n",
      "|    n_updates        | 974      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 18       |\n",
      "|    ep_rew_mean      | -66.2    |\n",
      "|    exploration_rate | 0.685    |\n",
      "| time/               |          |\n",
      "|    episodes         | 292      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1966     |\n",
      "|    total_timesteps  | 4979     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.59     |\n",
      "|    n_updates        | 994      |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 18.1     |\n",
      "|    ep_rew_mean      | -65.6    |\n",
      "|    exploration_rate | 0.679    |\n",
      "| time/               |          |\n",
      "|    episodes         | 296      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 1993     |\n",
      "|    total_timesteps  | 5064     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.74     |\n",
      "|    n_updates        | 1015     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 18.2     |\n",
      "|    ep_rew_mean      | -65.2    |\n",
      "|    exploration_rate | 0.674    |\n",
      "| time/               |          |\n",
      "|    episodes         | 300      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2020     |\n",
      "|    total_timesteps  | 5141     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 7.51     |\n",
      "|    n_updates        | 1035     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 18.3     |\n",
      "|    ep_rew_mean      | -65.1    |\n",
      "|    exploration_rate | 0.67     |\n",
      "| time/               |          |\n",
      "|    episodes         | 304      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2046     |\n",
      "|    total_timesteps  | 5216     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.33     |\n",
      "|    n_updates        | 1053     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 18.4     |\n",
      "|    ep_rew_mean      | -64.7    |\n",
      "|    exploration_rate | 0.665    |\n",
      "| time/               |          |\n",
      "|    episodes         | 308      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2072     |\n",
      "|    total_timesteps  | 5295     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.63     |\n",
      "|    n_updates        | 1073     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 18.3     |\n",
      "|    ep_rew_mean      | -64.8    |\n",
      "|    exploration_rate | 0.661    |\n",
      "| time/               |          |\n",
      "|    episodes         | 312      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2097     |\n",
      "|    total_timesteps  | 5359     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.45     |\n",
      "|    n_updates        | 1089     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 18.5     |\n",
      "|    ep_rew_mean      | -64.5    |\n",
      "|    exploration_rate | 0.655    |\n",
      "| time/               |          |\n",
      "|    episodes         | 316      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2125     |\n",
      "|    total_timesteps  | 5440     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.587    |\n",
      "|    n_updates        | 1109     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 18.3     |\n",
      "|    ep_rew_mean      | -64.7    |\n",
      "|    exploration_rate | 0.651    |\n",
      "| time/               |          |\n",
      "|    episodes         | 320      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2151     |\n",
      "|    total_timesteps  | 5503     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.64     |\n",
      "|    n_updates        | 1125     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 18.5     |\n",
      "|    ep_rew_mean      | -64      |\n",
      "|    exploration_rate | 0.646    |\n",
      "| time/               |          |\n",
      "|    episodes         | 324      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2176     |\n",
      "|    total_timesteps  | 5589     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.22     |\n",
      "|    n_updates        | 1147     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 18.6     |\n",
      "|    ep_rew_mean      | -63.8    |\n",
      "|    exploration_rate | 0.642    |\n",
      "| time/               |          |\n",
      "|    episodes         | 328      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2203     |\n",
      "|    total_timesteps  | 5659     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.27     |\n",
      "|    n_updates        | 1164     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 18.8     |\n",
      "|    ep_rew_mean      | -62.9    |\n",
      "|    exploration_rate | 0.636    |\n",
      "| time/               |          |\n",
      "|    episodes         | 332      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2231     |\n",
      "|    total_timesteps  | 5752     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.02     |\n",
      "|    n_updates        | 1187     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 19       |\n",
      "|    ep_rew_mean      | -62.4    |\n",
      "|    exploration_rate | 0.631    |\n",
      "| time/               |          |\n",
      "|    episodes         | 336      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2257     |\n",
      "|    total_timesteps  | 5830     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.158    |\n",
      "|    n_updates        | 1207     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 18.9     |\n",
      "|    ep_rew_mean      | -62.6    |\n",
      "|    exploration_rate | 0.627    |\n",
      "| time/               |          |\n",
      "|    episodes         | 340      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2283     |\n",
      "|    total_timesteps  | 5891     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.841    |\n",
      "|    n_updates        | 1222     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 19.1     |\n",
      "|    ep_rew_mean      | -62      |\n",
      "|    exploration_rate | 0.621    |\n",
      "| time/               |          |\n",
      "|    episodes         | 344      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2311     |\n",
      "|    total_timesteps  | 5977     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.322    |\n",
      "|    n_updates        | 1244     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 19.1     |\n",
      "|    ep_rew_mean      | -62      |\n",
      "|    exploration_rate | 0.617    |\n",
      "| time/               |          |\n",
      "|    episodes         | 348      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2336     |\n",
      "|    total_timesteps  | 6047     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.533    |\n",
      "|    n_updates        | 1261     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 18.9     |\n",
      "|    ep_rew_mean      | -62.5    |\n",
      "|    exploration_rate | 0.613    |\n",
      "| time/               |          |\n",
      "|    episodes         | 352      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2361     |\n",
      "|    total_timesteps  | 6115     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.204    |\n",
      "|    n_updates        | 1278     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 19       |\n",
      "|    ep_rew_mean      | -62      |\n",
      "|    exploration_rate | 0.607    |\n",
      "| time/               |          |\n",
      "|    episodes         | 356      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2387     |\n",
      "|    total_timesteps  | 6199     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.848    |\n",
      "|    n_updates        | 1299     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 19.1     |\n",
      "|    ep_rew_mean      | -61.9    |\n",
      "|    exploration_rate | 0.603    |\n",
      "| time/               |          |\n",
      "|    episodes         | 360      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2412     |\n",
      "|    total_timesteps  | 6273     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.52     |\n",
      "|    n_updates        | 1318     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 19.2     |\n",
      "|    ep_rew_mean      | -61.2    |\n",
      "|    exploration_rate | 0.597    |\n",
      "| time/               |          |\n",
      "|    episodes         | 364      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2439     |\n",
      "|    total_timesteps  | 6368     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.37     |\n",
      "|    n_updates        | 1341     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 19.4     |\n",
      "|    ep_rew_mean      | -60.7    |\n",
      "|    exploration_rate | 0.591    |\n",
      "| time/               |          |\n",
      "|    episodes         | 368      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2465     |\n",
      "|    total_timesteps  | 6456     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.319    |\n",
      "|    n_updates        | 1363     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 19.5     |\n",
      "|    ep_rew_mean      | -60.3    |\n",
      "|    exploration_rate | 0.585    |\n",
      "| time/               |          |\n",
      "|    episodes         | 372      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2493     |\n",
      "|    total_timesteps  | 6545     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.905    |\n",
      "|    n_updates        | 1386     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 19.6     |\n",
      "|    ep_rew_mean      | -60      |\n",
      "|    exploration_rate | 0.58     |\n",
      "| time/               |          |\n",
      "|    episodes         | 376      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2520     |\n",
      "|    total_timesteps  | 6629     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 6.8      |\n",
      "|    n_updates        | 1407     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 19.9     |\n",
      "|    ep_rew_mean      | -58.9    |\n",
      "|    exploration_rate | 0.574    |\n",
      "| time/               |          |\n",
      "|    episodes         | 380      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2546     |\n",
      "|    total_timesteps  | 6722     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 7.88     |\n",
      "|    n_updates        | 1430     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 20       |\n",
      "|    ep_rew_mean      | -58.5    |\n",
      "|    exploration_rate | 0.568    |\n",
      "| time/               |          |\n",
      "|    episodes         | 384      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2576     |\n",
      "|    total_timesteps  | 6817     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.49     |\n",
      "|    n_updates        | 1454     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 20.1     |\n",
      "|    ep_rew_mean      | -57.9    |\n",
      "|    exploration_rate | 0.562    |\n",
      "| time/               |          |\n",
      "|    episodes         | 388      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2603     |\n",
      "|    total_timesteps  | 6912     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 10.1     |\n",
      "|    n_updates        | 1477     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 20.1     |\n",
      "|    ep_rew_mean      | -57.9    |\n",
      "|    exploration_rate | 0.557    |\n",
      "| time/               |          |\n",
      "|    episodes         | 392      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2629     |\n",
      "|    total_timesteps  | 6990     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.54     |\n",
      "|    n_updates        | 1497     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 20.1     |\n",
      "|    ep_rew_mean      | -57.9    |\n",
      "|    exploration_rate | 0.552    |\n",
      "| time/               |          |\n",
      "|    episodes         | 396      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2655     |\n",
      "|    total_timesteps  | 7073     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 6.3      |\n",
      "|    n_updates        | 1518     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 20.2     |\n",
      "|    ep_rew_mean      | -57.6    |\n",
      "|    exploration_rate | 0.547    |\n",
      "| time/               |          |\n",
      "|    episodes         | 400      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2682     |\n",
      "|    total_timesteps  | 7159     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.19     |\n",
      "|    n_updates        | 1539     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 20.2     |\n",
      "|    ep_rew_mean      | -57.4    |\n",
      "|    exploration_rate | 0.542    |\n",
      "| time/               |          |\n",
      "|    episodes         | 404      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2708     |\n",
      "|    total_timesteps  | 7239     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.06     |\n",
      "|    n_updates        | 1559     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 20.2     |\n",
      "|    ep_rew_mean      | -57.6    |\n",
      "|    exploration_rate | 0.537    |\n",
      "| time/               |          |\n",
      "|    episodes         | 408      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2734     |\n",
      "|    total_timesteps  | 7312     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.355    |\n",
      "|    n_updates        | 1577     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 20.5     |\n",
      "|    ep_rew_mean      | -56.6    |\n",
      "|    exploration_rate | 0.531    |\n",
      "| time/               |          |\n",
      "|    episodes         | 412      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2761     |\n",
      "|    total_timesteps  | 7407     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.311    |\n",
      "|    n_updates        | 1601     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 20.7     |\n",
      "|    ep_rew_mean      | -55.9    |\n",
      "|    exploration_rate | 0.525    |\n",
      "| time/               |          |\n",
      "|    episodes         | 416      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2786     |\n",
      "|    total_timesteps  | 7507     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.08     |\n",
      "|    n_updates        | 1626     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 20.9     |\n",
      "|    ep_rew_mean      | -55.2    |\n",
      "|    exploration_rate | 0.519    |\n",
      "| time/               |          |\n",
      "|    episodes         | 420      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2814     |\n",
      "|    total_timesteps  | 7590     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.347    |\n",
      "|    n_updates        | 1647     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 20.9     |\n",
      "|    ep_rew_mean      | -55.3    |\n",
      "|    exploration_rate | 0.514    |\n",
      "| time/               |          |\n",
      "|    episodes         | 424      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2839     |\n",
      "|    total_timesteps  | 7674     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.66     |\n",
      "|    n_updates        | 1668     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 21.1     |\n",
      "|    ep_rew_mean      | -54.4    |\n",
      "|    exploration_rate | 0.508    |\n",
      "| time/               |          |\n",
      "|    episodes         | 428      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2865     |\n",
      "|    total_timesteps  | 7768     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0851   |\n",
      "|    n_updates        | 1691     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 21.2     |\n",
      "|    ep_rew_mean      | -54      |\n",
      "|    exploration_rate | 0.501    |\n",
      "| time/               |          |\n",
      "|    episodes         | 432      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2891     |\n",
      "|    total_timesteps  | 7873     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.09     |\n",
      "|    n_updates        | 1718     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 21.3     |\n",
      "|    ep_rew_mean      | -53.5    |\n",
      "|    exploration_rate | 0.496    |\n",
      "| time/               |          |\n",
      "|    episodes         | 436      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2917     |\n",
      "|    total_timesteps  | 7961     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 5.37     |\n",
      "|    n_updates        | 1740     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 21.6     |\n",
      "|    ep_rew_mean      | -52.4    |\n",
      "|    exploration_rate | 0.49     |\n",
      "| time/               |          |\n",
      "|    episodes         | 440      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2942     |\n",
      "|    total_timesteps  | 8054     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.36     |\n",
      "|    n_updates        | 1763     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 21.6     |\n",
      "|    ep_rew_mean      | -52.4    |\n",
      "|    exploration_rate | 0.485    |\n",
      "| time/               |          |\n",
      "|    episodes         | 444      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2968     |\n",
      "|    total_timesteps  | 8136     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.54     |\n",
      "|    n_updates        | 1783     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 21.8     |\n",
      "|    ep_rew_mean      | -51.9    |\n",
      "|    exploration_rate | 0.479    |\n",
      "| time/               |          |\n",
      "|    episodes         | 448      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 2993     |\n",
      "|    total_timesteps  | 8223     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.287    |\n",
      "|    n_updates        | 1805     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 22       |\n",
      "|    ep_rew_mean      | -51      |\n",
      "|    exploration_rate | 0.473    |\n",
      "| time/               |          |\n",
      "|    episodes         | 452      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3020     |\n",
      "|    total_timesteps  | 8319     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.745    |\n",
      "|    n_updates        | 1829     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 22.1     |\n",
      "|    ep_rew_mean      | -50.8    |\n",
      "|    exploration_rate | 0.467    |\n",
      "| time/               |          |\n",
      "|    episodes         | 456      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3045     |\n",
      "|    total_timesteps  | 8409     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.3      |\n",
      "|    n_updates        | 1852     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 22.4     |\n",
      "|    ep_rew_mean      | -49.8    |\n",
      "|    exploration_rate | 0.461    |\n",
      "| time/               |          |\n",
      "|    episodes         | 460      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3070     |\n",
      "|    total_timesteps  | 8511     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0615   |\n",
      "|    n_updates        | 1877     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 22.8     |\n",
      "|    ep_rew_mean      | -49      |\n",
      "|    exploration_rate | 0.453    |\n",
      "| time/               |          |\n",
      "|    episodes         | 464      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3106     |\n",
      "|    total_timesteps  | 8643     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.46     |\n",
      "|    n_updates        | 1910     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 22.9     |\n",
      "|    ep_rew_mean      | -48.5    |\n",
      "|    exploration_rate | 0.446    |\n",
      "| time/               |          |\n",
      "|    episodes         | 468      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3134     |\n",
      "|    total_timesteps  | 8748     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.178    |\n",
      "|    n_updates        | 1936     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 23.1     |\n",
      "|    ep_rew_mean      | -47.8    |\n",
      "|    exploration_rate | 0.439    |\n",
      "| time/               |          |\n",
      "|    episodes         | 472      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3161     |\n",
      "|    total_timesteps  | 8856     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.797    |\n",
      "|    n_updates        | 1963     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 23.2     |\n",
      "|    ep_rew_mean      | -47.3    |\n",
      "|    exploration_rate | 0.433    |\n",
      "| time/               |          |\n",
      "|    episodes         | 476      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3186     |\n",
      "|    total_timesteps  | 8951     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.34     |\n",
      "|    n_updates        | 1987     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 23.2     |\n",
      "|    ep_rew_mean      | -47.2    |\n",
      "|    exploration_rate | 0.427    |\n",
      "| time/               |          |\n",
      "|    episodes         | 480      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3212     |\n",
      "|    total_timesteps  | 9047     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0437   |\n",
      "|    n_updates        | 2011     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 23.3     |\n",
      "|    ep_rew_mean      | -46.8    |\n",
      "|    exploration_rate | 0.421    |\n",
      "| time/               |          |\n",
      "|    episodes         | 484      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3238     |\n",
      "|    total_timesteps  | 9149     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.354    |\n",
      "|    n_updates        | 2037     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 23.3     |\n",
      "|    ep_rew_mean      | -46.8    |\n",
      "|    exploration_rate | 0.415    |\n",
      "| time/               |          |\n",
      "|    episodes         | 488      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3264     |\n",
      "|    total_timesteps  | 9242     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.789    |\n",
      "|    n_updates        | 2060     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 23.6     |\n",
      "|    ep_rew_mean      | -45.7    |\n",
      "|    exploration_rate | 0.408    |\n",
      "| time/               |          |\n",
      "|    episodes         | 492      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3291     |\n",
      "|    total_timesteps  | 9355     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.01     |\n",
      "|    n_updates        | 2088     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 23.7     |\n",
      "|    ep_rew_mean      | -45.5    |\n",
      "|    exploration_rate | 0.402    |\n",
      "| time/               |          |\n",
      "|    episodes         | 496      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3319     |\n",
      "|    total_timesteps  | 9446     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4        |\n",
      "|    n_updates        | 2111     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 23.9     |\n",
      "|    ep_rew_mean      | -45      |\n",
      "|    exploration_rate | 0.396    |\n",
      "| time/               |          |\n",
      "|    episodes         | 500      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3344     |\n",
      "|    total_timesteps  | 9544     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.581    |\n",
      "|    n_updates        | 2135     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 24.1     |\n",
      "|    ep_rew_mean      | -44.2    |\n",
      "|    exploration_rate | 0.389    |\n",
      "| time/               |          |\n",
      "|    episodes         | 504      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3370     |\n",
      "|    total_timesteps  | 9646     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0605   |\n",
      "|    n_updates        | 2161     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 24.5     |\n",
      "|    ep_rew_mean      | -42.9    |\n",
      "|    exploration_rate | 0.382    |\n",
      "| time/               |          |\n",
      "|    episodes         | 508      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3399     |\n",
      "|    total_timesteps  | 9762     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.558    |\n",
      "|    n_updates        | 2190     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 24.4     |\n",
      "|    ep_rew_mean      | -43      |\n",
      "|    exploration_rate | 0.376    |\n",
      "| time/               |          |\n",
      "|    episodes         | 512      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3425     |\n",
      "|    total_timesteps  | 9851     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.382    |\n",
      "|    n_updates        | 2212     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 24.6     |\n",
      "|    ep_rew_mean      | -42.8    |\n",
      "|    exploration_rate | 0.369    |\n",
      "| time/               |          |\n",
      "|    episodes         | 516      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3452     |\n",
      "|    total_timesteps  | 9963     |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.65     |\n",
      "|    n_updates        | 2240     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 24.9     |\n",
      "|    ep_rew_mean      | -41.5    |\n",
      "|    exploration_rate | 0.362    |\n",
      "| time/               |          |\n",
      "|    episodes         | 520      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3480     |\n",
      "|    total_timesteps  | 10081    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0767   |\n",
      "|    n_updates        | 2270     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 25.1     |\n",
      "|    ep_rew_mean      | -41.1    |\n",
      "|    exploration_rate | 0.355    |\n",
      "| time/               |          |\n",
      "|    episodes         | 524      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3505     |\n",
      "|    total_timesteps  | 10179    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 5.44     |\n",
      "|    n_updates        | 2294     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 25.2     |\n",
      "|    ep_rew_mean      | -40.7    |\n",
      "|    exploration_rate | 0.348    |\n",
      "| time/               |          |\n",
      "|    episodes         | 528      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3533     |\n",
      "|    total_timesteps  | 10288    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.141    |\n",
      "|    n_updates        | 2321     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 25.2     |\n",
      "|    ep_rew_mean      | -40.9    |\n",
      "|    exploration_rate | 0.342    |\n",
      "| time/               |          |\n",
      "|    episodes         | 532      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3560     |\n",
      "|    total_timesteps  | 10389    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.352    |\n",
      "|    n_updates        | 2347     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 25.5     |\n",
      "|    ep_rew_mean      | -39.9    |\n",
      "|    exploration_rate | 0.334    |\n",
      "| time/               |          |\n",
      "|    episodes         | 536      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3588     |\n",
      "|    total_timesteps  | 10509    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.14     |\n",
      "|    n_updates        | 2377     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 25.4     |\n",
      "|    ep_rew_mean      | -40.2    |\n",
      "|    exploration_rate | 0.329    |\n",
      "| time/               |          |\n",
      "|    episodes         | 540      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3613     |\n",
      "|    total_timesteps  | 10594    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.98     |\n",
      "|    n_updates        | 2398     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 25.7     |\n",
      "|    ep_rew_mean      | -39.3    |\n",
      "|    exploration_rate | 0.322    |\n",
      "| time/               |          |\n",
      "|    episodes         | 544      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3639     |\n",
      "|    total_timesteps  | 10705    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 6.18     |\n",
      "|    n_updates        | 2426     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 26       |\n",
      "|    ep_rew_mean      | -38.3    |\n",
      "|    exploration_rate | 0.315    |\n",
      "| time/               |          |\n",
      "|    episodes         | 548      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3664     |\n",
      "|    total_timesteps  | 10820    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 5.93     |\n",
      "|    n_updates        | 2454     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 26.2     |\n",
      "|    ep_rew_mean      | -37.4    |\n",
      "|    exploration_rate | 0.307    |\n",
      "| time/               |          |\n",
      "|    episodes         | 552      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3691     |\n",
      "|    total_timesteps  | 10944    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.74     |\n",
      "|    n_updates        | 2485     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 26.6     |\n",
      "|    ep_rew_mean      | -36.4    |\n",
      "|    exploration_rate | 0.299    |\n",
      "| time/               |          |\n",
      "|    episodes         | 556      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3719     |\n",
      "|    total_timesteps  | 11067    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 6.07     |\n",
      "|    n_updates        | 2516     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 26.9     |\n",
      "|    ep_rew_mean      | -35.3    |\n",
      "|    exploration_rate | 0.291    |\n",
      "| time/               |          |\n",
      "|    episodes         | 560      |\n",
      "|    fps              | 2        |\n",
      "|    time_elapsed     | 3744     |\n",
      "|    total_timesteps  | 11199    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.355    |\n",
      "|    n_updates        | 2549     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 26.9     |\n",
      "|    ep_rew_mean      | -34.9    |\n",
      "|    exploration_rate | 0.282    |\n",
      "| time/               |          |\n",
      "|    episodes         | 564      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 3770     |\n",
      "|    total_timesteps  | 11329    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 8.41     |\n",
      "|    n_updates        | 2582     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 27.3     |\n",
      "|    ep_rew_mean      | -33.6    |\n",
      "|    exploration_rate | 0.273    |\n",
      "| time/               |          |\n",
      "|    episodes         | 568      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 3797     |\n",
      "|    total_timesteps  | 11474    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 6.33     |\n",
      "|    n_updates        | 2618     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 27.5     |\n",
      "|    ep_rew_mean      | -33      |\n",
      "|    exploration_rate | 0.265    |\n",
      "| time/               |          |\n",
      "|    episodes         | 572      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 3827     |\n",
      "|    total_timesteps  | 11604    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.32     |\n",
      "|    n_updates        | 2650     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 27.8     |\n",
      "|    ep_rew_mean      | -32      |\n",
      "|    exploration_rate | 0.257    |\n",
      "| time/               |          |\n",
      "|    episodes         | 576      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 3853     |\n",
      "|    total_timesteps  | 11729    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.23     |\n",
      "|    n_updates        | 2682     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 27.8     |\n",
      "|    ep_rew_mean      | -31.9    |\n",
      "|    exploration_rate | 0.251    |\n",
      "| time/               |          |\n",
      "|    episodes         | 580      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 3879     |\n",
      "|    total_timesteps  | 11826    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.17     |\n",
      "|    n_updates        | 2706     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 28.1     |\n",
      "|    ep_rew_mean      | -31      |\n",
      "|    exploration_rate | 0.243    |\n",
      "| time/               |          |\n",
      "|    episodes         | 584      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 3908     |\n",
      "|    total_timesteps  | 11958    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 7.73     |\n",
      "|    n_updates        | 2739     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 28.6     |\n",
      "|    ep_rew_mean      | -29.4    |\n",
      "|    exploration_rate | 0.233    |\n",
      "| time/               |          |\n",
      "|    episodes         | 588      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 3939     |\n",
      "|    total_timesteps  | 12105    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.09     |\n",
      "|    n_updates        | 2776     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 29       |\n",
      "|    ep_rew_mean      | -28.4    |\n",
      "|    exploration_rate | 0.224    |\n",
      "| time/               |          |\n",
      "|    episodes         | 592      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 3971     |\n",
      "|    total_timesteps  | 12255    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.374    |\n",
      "|    n_updates        | 2813     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 29.6     |\n",
      "|    ep_rew_mean      | -26.6    |\n",
      "|    exploration_rate | 0.215    |\n",
      "| time/               |          |\n",
      "|    episodes         | 596      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4002     |\n",
      "|    total_timesteps  | 12402    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.89     |\n",
      "|    n_updates        | 2850     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 30       |\n",
      "|    ep_rew_mean      | -25.4    |\n",
      "|    exploration_rate | 0.206    |\n",
      "| time/               |          |\n",
      "|    episodes         | 600      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4031     |\n",
      "|    total_timesteps  | 12541    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.679    |\n",
      "|    n_updates        | 2885     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 30.5     |\n",
      "|    ep_rew_mean      | -23.8    |\n",
      "|    exploration_rate | 0.196    |\n",
      "| time/               |          |\n",
      "|    episodes         | 604      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4063     |\n",
      "|    total_timesteps  | 12700    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.34     |\n",
      "|    n_updates        | 2924     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 30.9     |\n",
      "|    ep_rew_mean      | -22.7    |\n",
      "|    exploration_rate | 0.186    |\n",
      "| time/               |          |\n",
      "|    episodes         | 608      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4094     |\n",
      "|    total_timesteps  | 12849    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.418    |\n",
      "|    n_updates        | 2962     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 31.8     |\n",
      "|    ep_rew_mean      | -20.1    |\n",
      "|    exploration_rate | 0.175    |\n",
      "| time/               |          |\n",
      "|    episodes         | 612      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4128     |\n",
      "|    total_timesteps  | 13028    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.585    |\n",
      "|    n_updates        | 3006     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 32.8     |\n",
      "|    ep_rew_mean      | -17.2    |\n",
      "|    exploration_rate | 0.161    |\n",
      "| time/               |          |\n",
      "|    episodes         | 616      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4169     |\n",
      "|    total_timesteps  | 13242    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.96     |\n",
      "|    n_updates        | 3060     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 33.3     |\n",
      "|    ep_rew_mean      | -15.8    |\n",
      "|    exploration_rate | 0.151    |\n",
      "| time/               |          |\n",
      "|    episodes         | 620      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4201     |\n",
      "|    total_timesteps  | 13408    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 6        |\n",
      "|    n_updates        | 3101     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 34       |\n",
      "|    ep_rew_mean      | -13.6    |\n",
      "|    exploration_rate | 0.14     |\n",
      "| time/               |          |\n",
      "|    episodes         | 624      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4233     |\n",
      "|    total_timesteps  | 13577    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 7.87     |\n",
      "|    n_updates        | 3144     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 34.8     |\n",
      "|    ep_rew_mean      | -11.1    |\n",
      "|    exploration_rate | 0.128    |\n",
      "| time/               |          |\n",
      "|    episodes         | 628      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4266     |\n",
      "|    total_timesteps  | 13768    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 5.99     |\n",
      "|    n_updates        | 3191     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 35.8     |\n",
      "|    ep_rew_mean      | -8.01    |\n",
      "|    exploration_rate | 0.115    |\n",
      "| time/               |          |\n",
      "|    episodes         | 632      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4301     |\n",
      "|    total_timesteps  | 13970    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.02     |\n",
      "|    n_updates        | 3242     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 36.6     |\n",
      "|    ep_rew_mean      | -5.47    |\n",
      "|    exploration_rate | 0.102    |\n",
      "| time/               |          |\n",
      "|    episodes         | 636      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4337     |\n",
      "|    total_timesteps  | 14174    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.22     |\n",
      "|    n_updates        | 3293     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 38       |\n",
      "|    ep_rew_mean      | -1.56    |\n",
      "|    exploration_rate | 0.0886   |\n",
      "| time/               |          |\n",
      "|    episodes         | 640      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4375     |\n",
      "|    total_timesteps  | 14390    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 6.12     |\n",
      "|    n_updates        | 3347     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 39.9     |\n",
      "|    ep_rew_mean      | 4.05     |\n",
      "|    exploration_rate | 0.0696   |\n",
      "| time/               |          |\n",
      "|    episodes         | 644      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4420     |\n",
      "|    total_timesteps  | 14690    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 7.58     |\n",
      "|    n_updates        | 3422     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 41.6     |\n",
      "|    ep_rew_mean      | 9.22     |\n",
      "|    exploration_rate | 0.051    |\n",
      "| time/               |          |\n",
      "|    episodes         | 648      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4466     |\n",
      "|    total_timesteps  | 14984    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.706    |\n",
      "|    n_updates        | 3495     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 43.1     |\n",
      "|    ep_rew_mean      | 13.3     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 652      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4509     |\n",
      "|    total_timesteps  | 15250    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.311    |\n",
      "|    n_updates        | 3562     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 44.9     |\n",
      "|    ep_rew_mean      | 18.6     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 656      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4555     |\n",
      "|    total_timesteps  | 15554    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.842    |\n",
      "|    n_updates        | 3638     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 46.6     |\n",
      "|    ep_rew_mean      | 23.6     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 660      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4603     |\n",
      "|    total_timesteps  | 15860    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 5.45     |\n",
      "|    n_updates        | 3714     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 48.8     |\n",
      "|    ep_rew_mean      | 29.9     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 664      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4655     |\n",
      "|    total_timesteps  | 16210    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.58     |\n",
      "|    n_updates        | 3802     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 50.3     |\n",
      "|    ep_rew_mean      | 33.8     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 668      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4705     |\n",
      "|    total_timesteps  | 16501    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.28     |\n",
      "|    n_updates        | 3875     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 51.7     |\n",
      "|    ep_rew_mean      | 38.2     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 672      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4747     |\n",
      "|    total_timesteps  | 16777    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.35     |\n",
      "|    n_updates        | 3944     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 54       |\n",
      "|    ep_rew_mean      | 44.7     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 676      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4802     |\n",
      "|    total_timesteps  | 17130    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.24     |\n",
      "|    n_updates        | 4032     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 55.4     |\n",
      "|    ep_rew_mean      | 48.9     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 680      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4842     |\n",
      "|    total_timesteps  | 17370    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 10.7     |\n",
      "|    n_updates        | 4092     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 56.3     |\n",
      "|    ep_rew_mean      | 51.5     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 684      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4879     |\n",
      "|    total_timesteps  | 17586    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.28     |\n",
      "|    n_updates        | 4146     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 58       |\n",
      "|    ep_rew_mean      | 56.6     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 688      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4927     |\n",
      "|    total_timesteps  | 17907    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.183    |\n",
      "|    n_updates        | 4226     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 58.5     |\n",
      "|    ep_rew_mean      | 58.2     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 692      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 4961     |\n",
      "|    total_timesteps  | 18106    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.15     |\n",
      "|    n_updates        | 4276     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 59.9     |\n",
      "|    ep_rew_mean      | 62.4     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 696      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 5005     |\n",
      "|    total_timesteps  | 18394    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.11     |\n",
      "|    n_updates        | 4348     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 61.7     |\n",
      "|    ep_rew_mean      | 67.7     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 700      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 5054     |\n",
      "|    total_timesteps  | 18710    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.32     |\n",
      "|    n_updates        | 4427     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 62.7     |\n",
      "|    ep_rew_mean      | 70.8     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 704      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 5093     |\n",
      "|    total_timesteps  | 18969    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.158    |\n",
      "|    n_updates        | 4492     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 63.7     |\n",
      "|    ep_rew_mean      | 74       |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 708      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 5128     |\n",
      "|    total_timesteps  | 19217    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.725    |\n",
      "|    n_updates        | 4554     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 64.3     |\n",
      "|    ep_rew_mean      | 75.9     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 712      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 5166     |\n",
      "|    total_timesteps  | 19457    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.308    |\n",
      "|    n_updates        | 4614     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 64.7     |\n",
      "|    ep_rew_mean      | 77.5     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 716      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 5203     |\n",
      "|    total_timesteps  | 19711    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.15     |\n",
      "|    n_updates        | 4677     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 65.8     |\n",
      "|    ep_rew_mean      | 80.9     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 720      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 5247     |\n",
      "|    total_timesteps  | 19988    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.328    |\n",
      "|    n_updates        | 4746     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 67.5     |\n",
      "|    ep_rew_mean      | 86       |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 724      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 5297     |\n",
      "|    total_timesteps  | 20332    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.97     |\n",
      "|    n_updates        | 4832     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 69.1     |\n",
      "|    ep_rew_mean      | 90.5     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 728      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 5349     |\n",
      "|    total_timesteps  | 20678    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.09     |\n",
      "|    n_updates        | 4919     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 69.5     |\n",
      "|    ep_rew_mean      | 91.7     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 732      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 5386     |\n",
      "|    total_timesteps  | 20916    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.113    |\n",
      "|    n_updates        | 4978     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 71.6     |\n",
      "|    ep_rew_mean      | 97.7     |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 736      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 5449     |\n",
      "|    total_timesteps  | 21334    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.55     |\n",
      "|    n_updates        | 5083     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 74.8     |\n",
      "|    ep_rew_mean      | 107      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 740      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 5529     |\n",
      "|    total_timesteps  | 21867    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.26     |\n",
      "|    n_updates        | 5216     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 75.2     |\n",
      "|    ep_rew_mean      | 108      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 744      |\n",
      "|    fps              | 3        |\n",
      "|    time_elapsed     | 5580     |\n",
      "|    total_timesteps  | 22207    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.64     |\n",
      "|    n_updates        | 5301     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 77.3     |\n",
      "|    ep_rew_mean      | 114      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 748      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 5656     |\n",
      "|    total_timesteps  | 22719    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.5      |\n",
      "|    n_updates        | 5429     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 78.5     |\n",
      "|    ep_rew_mean      | 117      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 752      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 5717     |\n",
      "|    total_timesteps  | 23100    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.21     |\n",
      "|    n_updates        | 5524     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 78.2     |\n",
      "|    ep_rew_mean      | 116      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 756      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 5762     |\n",
      "|    total_timesteps  | 23377    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.2      |\n",
      "|    n_updates        | 5594     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 78.2     |\n",
      "|    ep_rew_mean      | 116      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 760      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 5812     |\n",
      "|    total_timesteps  | 23684    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.726    |\n",
      "|    n_updates        | 5670     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 78.6     |\n",
      "|    ep_rew_mean      | 117      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 764      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 5874     |\n",
      "|    total_timesteps  | 24074    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.85     |\n",
      "|    n_updates        | 5768     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 79.8     |\n",
      "|    ep_rew_mean      | 121      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 768      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 5935     |\n",
      "|    total_timesteps  | 24484    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.93     |\n",
      "|    n_updates        | 5870     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 80.5     |\n",
      "|    ep_rew_mean      | 122      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 772      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 5996     |\n",
      "|    total_timesteps  | 24829    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.751    |\n",
      "|    n_updates        | 5957     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 81.5     |\n",
      "|    ep_rew_mean      | 125      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 776      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 6068     |\n",
      "|    total_timesteps  | 25282    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.605    |\n",
      "|    n_updates        | 6070     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 81.3     |\n",
      "|    ep_rew_mean      | 124      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 780      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 6102     |\n",
      "|    total_timesteps  | 25502    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.55     |\n",
      "|    n_updates        | 6125     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 82.2     |\n",
      "|    ep_rew_mean      | 127      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 784      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 6148     |\n",
      "|    total_timesteps  | 25810    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2        |\n",
      "|    n_updates        | 6202     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 83.8     |\n",
      "|    ep_rew_mean      | 131      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 788      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 6217     |\n",
      "|    total_timesteps  | 26290    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.24     |\n",
      "|    n_updates        | 6322     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86       |\n",
      "|    ep_rew_mean      | 137      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 792      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 6281     |\n",
      "|    total_timesteps  | 26709    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.76     |\n",
      "|    n_updates        | 6427     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.1     |\n",
      "|    ep_rew_mean      | 135      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 796      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 6312     |\n",
      "|    total_timesteps  | 26903    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.28     |\n",
      "|    n_updates        | 6475     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.2     |\n",
      "|    ep_rew_mean      | 135      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 800      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 6360     |\n",
      "|    total_timesteps  | 27229    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.09     |\n",
      "|    n_updates        | 6557     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.5     |\n",
      "|    ep_rew_mean      | 136      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 804      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 6404     |\n",
      "|    total_timesteps  | 27514    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.525    |\n",
      "|    n_updates        | 6628     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.8     |\n",
      "|    ep_rew_mean      | 136      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 808      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 6449     |\n",
      "|    total_timesteps  | 27794    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.54     |\n",
      "|    n_updates        | 6698     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87.7     |\n",
      "|    ep_rew_mean      | 141      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 812      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 6524     |\n",
      "|    total_timesteps  | 28229    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.163    |\n",
      "|    n_updates        | 6807     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.4     |\n",
      "|    ep_rew_mean      | 146      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 816      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 6584     |\n",
      "|    total_timesteps  | 28649    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.819    |\n",
      "|    n_updates        | 6912     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 90.2     |\n",
      "|    ep_rew_mean      | 148      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 820      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 6638     |\n",
      "|    total_timesteps  | 29005    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.184    |\n",
      "|    n_updates        | 7001     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 92       |\n",
      "|    ep_rew_mean      | 153      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 824      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 6717     |\n",
      "|    total_timesteps  | 29528    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.76     |\n",
      "|    n_updates        | 7131     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 92.1     |\n",
      "|    ep_rew_mean      | 153      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 828      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 6770     |\n",
      "|    total_timesteps  | 29889    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.593    |\n",
      "|    n_updates        | 7222     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 95.5     |\n",
      "|    ep_rew_mean      | 162      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 832      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 6857     |\n",
      "|    total_timesteps  | 30464    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.671    |\n",
      "|    n_updates        | 7365     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 96.7     |\n",
      "|    ep_rew_mean      | 165      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 836      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 6940     |\n",
      "|    total_timesteps  | 31002    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.6      |\n",
      "|    n_updates        | 7500     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 95.8     |\n",
      "|    ep_rew_mean      | 162      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 840      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 7011     |\n",
      "|    total_timesteps  | 31445    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.919    |\n",
      "|    n_updates        | 7611     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 96       |\n",
      "|    ep_rew_mean      | 163      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 844      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 7066     |\n",
      "|    total_timesteps  | 31802    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 5.42     |\n",
      "|    n_updates        | 7700     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 93.8     |\n",
      "|    ep_rew_mean      | 157      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 848      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 7114     |\n",
      "|    total_timesteps  | 32103    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.706    |\n",
      "|    n_updates        | 7775     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 94.4     |\n",
      "|    ep_rew_mean      | 158      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 852      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 7180     |\n",
      "|    total_timesteps  | 32536    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.844    |\n",
      "|    n_updates        | 7883     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 96.3     |\n",
      "|    ep_rew_mean      | 163      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 856      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 7257     |\n",
      "|    total_timesteps  | 33005    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.33     |\n",
      "|    n_updates        | 8001     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 96.8     |\n",
      "|    ep_rew_mean      | 165      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 860      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 7312     |\n",
      "|    total_timesteps  | 33368    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.159    |\n",
      "|    n_updates        | 8091     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 98.3     |\n",
      "|    ep_rew_mean      | 169      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 864      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 7391     |\n",
      "|    total_timesteps  | 33900    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 5.73     |\n",
      "|    n_updates        | 8224     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 98.8     |\n",
      "|    ep_rew_mean      | 170      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 868      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 7460     |\n",
      "|    total_timesteps  | 34361    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.81     |\n",
      "|    n_updates        | 8340     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 99.8     |\n",
      "|    ep_rew_mean      | 173      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 872      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 7531     |\n",
      "|    total_timesteps  | 34812    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.296    |\n",
      "|    n_updates        | 8452     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 99.5     |\n",
      "|    ep_rew_mean      | 173      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 876      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 7593     |\n",
      "|    total_timesteps  | 35229    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.14     |\n",
      "|    n_updates        | 8557     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 104      |\n",
      "|    ep_rew_mean      | 184      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 880      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 7687     |\n",
      "|    total_timesteps  | 35870    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.518    |\n",
      "|    n_updates        | 8717     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 105      |\n",
      "|    ep_rew_mean      | 187      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 884      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 7752     |\n",
      "|    total_timesteps  | 36298    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.279    |\n",
      "|    n_updates        | 8824     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 105      |\n",
      "|    ep_rew_mean      | 188      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 888      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 7826     |\n",
      "|    total_timesteps  | 36786    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.521    |\n",
      "|    n_updates        | 8946     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 106      |\n",
      "|    ep_rew_mean      | 191      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 892      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 7912     |\n",
      "|    total_timesteps  | 37324    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.565    |\n",
      "|    n_updates        | 9080     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 111      |\n",
      "|    ep_rew_mean      | 205      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 896      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 8018     |\n",
      "|    total_timesteps  | 38033    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.05     |\n",
      "|    n_updates        | 9258     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 113      |\n",
      "|    ep_rew_mean      | 210      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 900      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 8097     |\n",
      "|    total_timesteps  | 38557    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.95     |\n",
      "|    n_updates        | 9389     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 113      |\n",
      "|    ep_rew_mean      | 209      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 904      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 8134     |\n",
      "|    total_timesteps  | 38793    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.368    |\n",
      "|    n_updates        | 9448     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 113      |\n",
      "|    ep_rew_mean      | 210      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 908      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 8184     |\n",
      "|    total_timesteps  | 39115    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1        |\n",
      "|    n_updates        | 9528     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 114      |\n",
      "|    ep_rew_mean      | 211      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 912      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 8252     |\n",
      "|    total_timesteps  | 39585    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.167    |\n",
      "|    n_updates        | 9646     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 113      |\n",
      "|    ep_rew_mean      | 210      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 916      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 8316     |\n",
      "|    total_timesteps  | 39978    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.422    |\n",
      "|    n_updates        | 9744     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 113      |\n",
      "|    ep_rew_mean      | 209      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 920      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 8361     |\n",
      "|    total_timesteps  | 40285    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.241    |\n",
      "|    n_updates        | 9821     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 113      |\n",
      "|    ep_rew_mean      | 209      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 924      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 8441     |\n",
      "|    total_timesteps  | 40809    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.136    |\n",
      "|    n_updates        | 9952     |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 112      |\n",
      "|    ep_rew_mean      | 206      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 928      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 8478     |\n",
      "|    total_timesteps  | 41051    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.385    |\n",
      "|    n_updates        | 10012    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 109      |\n",
      "|    ep_rew_mean      | 198      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 932      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 8525     |\n",
      "|    total_timesteps  | 41333    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.66     |\n",
      "|    n_updates        | 10083    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 107      |\n",
      "|    ep_rew_mean      | 194      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 936      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 8581     |\n",
      "|    total_timesteps  | 41719    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.38     |\n",
      "|    n_updates        | 10179    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 105      |\n",
      "|    ep_rew_mean      | 188      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 940      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 8614     |\n",
      "|    total_timesteps  | 41930    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.15     |\n",
      "|    n_updates        | 10232    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 106      |\n",
      "|    ep_rew_mean      | 190      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 944      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 8687     |\n",
      "|    total_timesteps  | 42381    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.649    |\n",
      "|    n_updates        | 10345    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 106      |\n",
      "|    ep_rew_mean      | 191      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 948      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 8745     |\n",
      "|    total_timesteps  | 42726    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.739    |\n",
      "|    n_updates        | 10431    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 107      |\n",
      "|    ep_rew_mean      | 193      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 952      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 8818     |\n",
      "|    total_timesteps  | 43228    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.81     |\n",
      "|    n_updates        | 10556    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 105      |\n",
      "|    ep_rew_mean      | 187      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 956      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 8857     |\n",
      "|    total_timesteps  | 43478    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.319    |\n",
      "|    n_updates        | 10619    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 106      |\n",
      "|    ep_rew_mean      | 189      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 960      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 8924     |\n",
      "|    total_timesteps  | 43920    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.06     |\n",
      "|    n_updates        | 10729    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 104      |\n",
      "|    ep_rew_mean      | 185      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 964      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 8978     |\n",
      "|    total_timesteps  | 44294    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.12     |\n",
      "|    n_updates        | 10823    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 102      |\n",
      "|    ep_rew_mean      | 180      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 968      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 9021     |\n",
      "|    total_timesteps  | 44565    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.09     |\n",
      "|    n_updates        | 10891    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 99.6     |\n",
      "|    ep_rew_mean      | 173      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 972      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 9055     |\n",
      "|    total_timesteps  | 44774    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.623    |\n",
      "|    n_updates        | 10943    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 98.7     |\n",
      "|    ep_rew_mean      | 171      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 976      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 9103     |\n",
      "|    total_timesteps  | 45103    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.199    |\n",
      "|    n_updates        | 11025    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 97.3     |\n",
      "|    ep_rew_mean      | 167      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 980      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 9179     |\n",
      "|    total_timesteps  | 45603    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.19     |\n",
      "|    n_updates        | 11150    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 97.4     |\n",
      "|    ep_rew_mean      | 167      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 984      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 9249     |\n",
      "|    total_timesteps  | 46034    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.224    |\n",
      "|    n_updates        | 11258    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 94.7     |\n",
      "|    ep_rew_mean      | 160      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 988      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 9282     |\n",
      "|    total_timesteps  | 46258    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.254    |\n",
      "|    n_updates        | 11314    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 94.7     |\n",
      "|    ep_rew_mean      | 160      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 992      |\n",
      "|    fps              | 4        |\n",
      "|    time_elapsed     | 9361     |\n",
      "|    total_timesteps  | 46797    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.97     |\n",
      "|    n_updates        | 11449    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 92.4     |\n",
      "|    ep_rew_mean      | 153      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 996      |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 9435     |\n",
      "|    total_timesteps  | 47273    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.53     |\n",
      "|    n_updates        | 11568    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 91.5     |\n",
      "|    ep_rew_mean      | 151      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1000     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 9501     |\n",
      "|    total_timesteps  | 47710    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.07     |\n",
      "|    n_updates        | 11677    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 93.3     |\n",
      "|    ep_rew_mean      | 156      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1004     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 9564     |\n",
      "|    total_timesteps  | 48125    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.247    |\n",
      "|    n_updates        | 11781    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 95.5     |\n",
      "|    ep_rew_mean      | 162      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1008     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 9647     |\n",
      "|    total_timesteps  | 48668    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.17     |\n",
      "|    n_updates        | 11916    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 94.2     |\n",
      "|    ep_rew_mean      | 158      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1012     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 9701     |\n",
      "|    total_timesteps  | 49009    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.33     |\n",
      "|    n_updates        | 12002    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 94.1     |\n",
      "|    ep_rew_mean      | 157      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1016     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 9767     |\n",
      "|    total_timesteps  | 49390    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.252    |\n",
      "|    n_updates        | 12097    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 94.6     |\n",
      "|    ep_rew_mean      | 158      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1020     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 9820     |\n",
      "|    total_timesteps  | 49744    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.169    |\n",
      "|    n_updates        | 12185    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 93.4     |\n",
      "|    ep_rew_mean      | 155      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1024     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 9884     |\n",
      "|    total_timesteps  | 50153    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.401    |\n",
      "|    n_updates        | 12288    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 94.9     |\n",
      "|    ep_rew_mean      | 159      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1028     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 9947     |\n",
      "|    total_timesteps  | 50541    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.17     |\n",
      "|    n_updates        | 12385    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 95.7     |\n",
      "|    ep_rew_mean      | 161      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1032     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 10006    |\n",
      "|    total_timesteps  | 50899    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.555    |\n",
      "|    n_updates        | 12474    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 95.1     |\n",
      "|    ep_rew_mean      | 159      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1036     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 10056    |\n",
      "|    total_timesteps  | 51229    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.968    |\n",
      "|    n_updates        | 12557    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 99.5     |\n",
      "|    ep_rew_mean      | 171      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1040     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 10156    |\n",
      "|    total_timesteps  | 51885    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.643    |\n",
      "|    n_updates        | 12721    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 97.9     |\n",
      "|    ep_rew_mean      | 167      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1044     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 10208    |\n",
      "|    total_timesteps  | 52167    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.183    |\n",
      "|    n_updates        | 12791    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 98.3     |\n",
      "|    ep_rew_mean      | 168      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1048     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 10271    |\n",
      "|    total_timesteps  | 52555    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.572    |\n",
      "|    n_updates        | 12888    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 97.3     |\n",
      "|    ep_rew_mean      | 165      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1052     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 10333    |\n",
      "|    total_timesteps  | 52960    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.51     |\n",
      "|    n_updates        | 12989    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 97.4     |\n",
      "|    ep_rew_mean      | 166      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1056     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 10371    |\n",
      "|    total_timesteps  | 53221    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.206    |\n",
      "|    n_updates        | 13055    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 95.9     |\n",
      "|    ep_rew_mean      | 161      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1060     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 10420    |\n",
      "|    total_timesteps  | 53511    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.35     |\n",
      "|    n_updates        | 13127    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 94.8     |\n",
      "|    ep_rew_mean      | 158      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1064     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 10461    |\n",
      "|    total_timesteps  | 53774    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.37     |\n",
      "|    n_updates        | 13193    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 94.7     |\n",
      "|    ep_rew_mean      | 158      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1068     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 10501    |\n",
      "|    total_timesteps  | 54030    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.96     |\n",
      "|    n_updates        | 13257    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 95.2     |\n",
      "|    ep_rew_mean      | 159      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1072     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 10546    |\n",
      "|    total_timesteps  | 54297    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.95     |\n",
      "|    n_updates        | 13324    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 95.5     |\n",
      "|    ep_rew_mean      | 160      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1076     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 10602    |\n",
      "|    total_timesteps  | 54658    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.245    |\n",
      "|    n_updates        | 13414    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 93.9     |\n",
      "|    ep_rew_mean      | 155      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1080     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 10663    |\n",
      "|    total_timesteps  | 54993    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 5.95     |\n",
      "|    n_updates        | 13498    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 93.2     |\n",
      "|    ep_rew_mean      | 153      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1084     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 10719    |\n",
      "|    total_timesteps  | 55349    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.173    |\n",
      "|    n_updates        | 13587    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 93.5     |\n",
      "|    ep_rew_mean      | 154      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1088     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 10761    |\n",
      "|    total_timesteps  | 55607    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.91     |\n",
      "|    n_updates        | 13651    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 90.9     |\n",
      "|    ep_rew_mean      | 147      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1092     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 10806    |\n",
      "|    total_timesteps  | 55884    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.427    |\n",
      "|    n_updates        | 13720    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.9     |\n",
      "|    ep_rew_mean      | 144      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1096     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 10868    |\n",
      "|    total_timesteps  | 56261    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.94     |\n",
      "|    n_updates        | 13815    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.1     |\n",
      "|    ep_rew_mean      | 142      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1100     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 10924    |\n",
      "|    total_timesteps  | 56618    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.701    |\n",
      "|    n_updates        | 13904    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.2     |\n",
      "|    ep_rew_mean      | 142      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1104     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11000    |\n",
      "|    total_timesteps  | 57047    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.326    |\n",
      "|    n_updates        | 14011    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 88       |\n",
      "|    ep_rew_mean      | 138      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1108     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11066    |\n",
      "|    total_timesteps  | 57463    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.243    |\n",
      "|    n_updates        | 14115    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87.8     |\n",
      "|    ep_rew_mean      | 138      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1112     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11118    |\n",
      "|    total_timesteps  | 57793    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.841    |\n",
      "|    n_updates        | 14198    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86.8     |\n",
      "|    ep_rew_mean      | 136      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1116     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11162    |\n",
      "|    total_timesteps  | 58065    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.361    |\n",
      "|    n_updates        | 14266    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86.2     |\n",
      "|    ep_rew_mean      | 134      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1120     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11206    |\n",
      "|    total_timesteps  | 58359    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.253    |\n",
      "|    n_updates        | 14339    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 84.7     |\n",
      "|    ep_rew_mean      | 130      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1124     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11251    |\n",
      "|    total_timesteps  | 58626    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.402    |\n",
      "|    n_updates        | 14406    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.5     |\n",
      "|    ep_rew_mean      | 132      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1128     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11323    |\n",
      "|    total_timesteps  | 59096    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.41     |\n",
      "|    n_updates        | 14523    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.8     |\n",
      "|    ep_rew_mean      | 133      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1132     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11380    |\n",
      "|    total_timesteps  | 59483    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.443    |\n",
      "|    n_updates        | 14620    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 84.9     |\n",
      "|    ep_rew_mean      | 131      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1136     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11416    |\n",
      "|    total_timesteps  | 59718    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.82     |\n",
      "|    n_updates        | 14679    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 83.5     |\n",
      "|    ep_rew_mean      | 127      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1140     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11492    |\n",
      "|    total_timesteps  | 60240    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.781    |\n",
      "|    n_updates        | 14809    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 83.7     |\n",
      "|    ep_rew_mean      | 128      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1144     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11536    |\n",
      "|    total_timesteps  | 60538    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.33     |\n",
      "|    n_updates        | 14884    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 82.8     |\n",
      "|    ep_rew_mean      | 126      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1148     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11584    |\n",
      "|    total_timesteps  | 60837    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.58     |\n",
      "|    n_updates        | 14959    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 82.5     |\n",
      "|    ep_rew_mean      | 125      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1152     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11636    |\n",
      "|    total_timesteps  | 61208    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.665    |\n",
      "|    n_updates        | 15051    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 84.9     |\n",
      "|    ep_rew_mean      | 132      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1156     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11710    |\n",
      "|    total_timesteps  | 61710    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.366    |\n",
      "|    n_updates        | 15177    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86.5     |\n",
      "|    ep_rew_mean      | 136      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1160     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11774    |\n",
      "|    total_timesteps  | 62156    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.04     |\n",
      "|    n_updates        | 15288    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86.5     |\n",
      "|    ep_rew_mean      | 136      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1164     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11814    |\n",
      "|    total_timesteps  | 62419    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.315    |\n",
      "|    n_updates        | 15354    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86.2     |\n",
      "|    ep_rew_mean      | 135      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1168     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11857    |\n",
      "|    total_timesteps  | 62646    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.331    |\n",
      "|    n_updates        | 15411    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86.9     |\n",
      "|    ep_rew_mean      | 137      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1172     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11923    |\n",
      "|    total_timesteps  | 62989    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.203    |\n",
      "|    n_updates        | 15497    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87       |\n",
      "|    ep_rew_mean      | 137      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1176     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 11980    |\n",
      "|    total_timesteps  | 63358    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.701    |\n",
      "|    n_updates        | 15589    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87.6     |\n",
      "|    ep_rew_mean      | 139      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1180     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12037    |\n",
      "|    total_timesteps  | 63755    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.293    |\n",
      "|    n_updates        | 15688    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87.9     |\n",
      "|    ep_rew_mean      | 140      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1184     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12096    |\n",
      "|    total_timesteps  | 64136    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.499    |\n",
      "|    n_updates        | 15783    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 90       |\n",
      "|    ep_rew_mean      | 146      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1188     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12166    |\n",
      "|    total_timesteps  | 64605    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.513    |\n",
      "|    n_updates        | 15901    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 91.1     |\n",
      "|    ep_rew_mean      | 149      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1192     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12223    |\n",
      "|    total_timesteps  | 64995    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.228    |\n",
      "|    n_updates        | 15998    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 91.8     |\n",
      "|    ep_rew_mean      | 151      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1196     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12291    |\n",
      "|    total_timesteps  | 65439    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.282    |\n",
      "|    n_updates        | 16109    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 90.6     |\n",
      "|    ep_rew_mean      | 148      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1200     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12327    |\n",
      "|    total_timesteps  | 65678    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.63     |\n",
      "|    n_updates        | 16169    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.5     |\n",
      "|    ep_rew_mean      | 145      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1204     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12374    |\n",
      "|    total_timesteps  | 65994    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.216    |\n",
      "|    n_updates        | 16248    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 88.6     |\n",
      "|    ep_rew_mean      | 143      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1208     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12424    |\n",
      "|    total_timesteps  | 66322    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.776    |\n",
      "|    n_updates        | 16330    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87.2     |\n",
      "|    ep_rew_mean      | 139      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1212     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12454    |\n",
      "|    total_timesteps  | 66510    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.894    |\n",
      "|    n_updates        | 16377    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87.8     |\n",
      "|    ep_rew_mean      | 141      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1216     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12509    |\n",
      "|    total_timesteps  | 66845    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.173    |\n",
      "|    n_updates        | 16461    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 88       |\n",
      "|    ep_rew_mean      | 141      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1220     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12558    |\n",
      "|    total_timesteps  | 67155    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.157    |\n",
      "|    n_updates        | 16538    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87.8     |\n",
      "|    ep_rew_mean      | 141      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1224     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12605    |\n",
      "|    total_timesteps  | 67411    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.469    |\n",
      "|    n_updates        | 16602    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.9     |\n",
      "|    ep_rew_mean      | 135      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1228     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12648    |\n",
      "|    total_timesteps  | 67682    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.361    |\n",
      "|    n_updates        | 16670    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.7     |\n",
      "|    ep_rew_mean      | 135      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1232     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12702    |\n",
      "|    total_timesteps  | 68049    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 5.65     |\n",
      "|    n_updates        | 16762    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.7     |\n",
      "|    ep_rew_mean      | 134      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1236     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12740    |\n",
      "|    total_timesteps  | 68284    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.22     |\n",
      "|    n_updates        | 16820    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 82.9     |\n",
      "|    ep_rew_mean      | 127      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1240     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12780    |\n",
      "|    total_timesteps  | 68532    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.224    |\n",
      "|    n_updates        | 16882    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 82.8     |\n",
      "|    ep_rew_mean      | 126      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1244     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12825    |\n",
      "|    total_timesteps  | 68814    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.702    |\n",
      "|    n_updates        | 16953    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 82.3     |\n",
      "|    ep_rew_mean      | 125      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1248     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12874    |\n",
      "|    total_timesteps  | 69069    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.217    |\n",
      "|    n_updates        | 17017    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 82.2     |\n",
      "|    ep_rew_mean      | 124      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1252     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12935    |\n",
      "|    total_timesteps  | 69424    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.15     |\n",
      "|    n_updates        | 17105    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 79.2     |\n",
      "|    ep_rew_mean      | 116      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1256     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 12966    |\n",
      "|    total_timesteps  | 69630    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.14     |\n",
      "|    n_updates        | 17157    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 78.3     |\n",
      "|    ep_rew_mean      | 113      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1260     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 13029    |\n",
      "|    total_timesteps  | 69989    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.686    |\n",
      "|    n_updates        | 17247    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 80.1     |\n",
      "|    ep_rew_mean      | 117      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1264     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 13104    |\n",
      "|    total_timesteps  | 70425    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 5.44     |\n",
      "|    n_updates        | 17356    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 82       |\n",
      "|    ep_rew_mean      | 122      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1268     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 13173    |\n",
      "|    total_timesteps  | 70849    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.422    |\n",
      "|    n_updates        | 17462    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 82.2     |\n",
      "|    ep_rew_mean      | 123      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1272     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 13233    |\n",
      "|    total_timesteps  | 71209    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.4      |\n",
      "|    n_updates        | 17552    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 81.7     |\n",
      "|    ep_rew_mean      | 122      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1276     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 13286    |\n",
      "|    total_timesteps  | 71529    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.437    |\n",
      "|    n_updates        | 17632    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 79.7     |\n",
      "|    ep_rew_mean      | 116      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1280     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 13319    |\n",
      "|    total_timesteps  | 71726    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.77     |\n",
      "|    n_updates        | 17681    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 80.7     |\n",
      "|    ep_rew_mean      | 118      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1284     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 13401    |\n",
      "|    total_timesteps  | 72201    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.61     |\n",
      "|    n_updates        | 17800    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 80.7     |\n",
      "|    ep_rew_mean      | 118      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1288     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 13481    |\n",
      "|    total_timesteps  | 72677    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2        |\n",
      "|    n_updates        | 17919    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 80       |\n",
      "|    ep_rew_mean      | 116      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1292     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 13543    |\n",
      "|    total_timesteps  | 72995    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.159    |\n",
      "|    n_updates        | 17998    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 81.7     |\n",
      "|    ep_rew_mean      | 120      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1296     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 13641    |\n",
      "|    total_timesteps  | 73611    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.89     |\n",
      "|    n_updates        | 18152    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.3     |\n",
      "|    ep_rew_mean      | 129      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1300     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 13736    |\n",
      "|    total_timesteps  | 74209    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.27     |\n",
      "|    n_updates        | 18302    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86.9     |\n",
      "|    ep_rew_mean      | 134      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1304     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 13809    |\n",
      "|    total_timesteps  | 74686    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.36     |\n",
      "|    n_updates        | 18421    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87.4     |\n",
      "|    ep_rew_mean      | 135      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1308     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 13867    |\n",
      "|    total_timesteps  | 75061    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.269    |\n",
      "|    n_updates        | 18515    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87.4     |\n",
      "|    ep_rew_mean      | 135      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1312     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 13900    |\n",
      "|    total_timesteps  | 75246    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.895    |\n",
      "|    n_updates        | 18561    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.3     |\n",
      "|    ep_rew_mean      | 139      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1316     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 13993    |\n",
      "|    total_timesteps  | 75775    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.335    |\n",
      "|    n_updates        | 18693    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 90.3     |\n",
      "|    ep_rew_mean      | 142      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1320     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 14059    |\n",
      "|    total_timesteps  | 76183    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.112    |\n",
      "|    n_updates        | 18795    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 90.3     |\n",
      "|    ep_rew_mean      | 142      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1324     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 14101    |\n",
      "|    total_timesteps  | 76440    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.31     |\n",
      "|    n_updates        | 18859    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 92.3     |\n",
      "|    ep_rew_mean      | 148      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1328     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 14171    |\n",
      "|    total_timesteps  | 76917    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0845   |\n",
      "|    n_updates        | 18979    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 92.2     |\n",
      "|    ep_rew_mean      | 147      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1332     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 14231    |\n",
      "|    total_timesteps  | 77269    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.227    |\n",
      "|    n_updates        | 19067    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 93       |\n",
      "|    ep_rew_mean      | 149      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1336     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 14279    |\n",
      "|    total_timesteps  | 77585    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.353    |\n",
      "|    n_updates        | 19146    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 94.6     |\n",
      "|    ep_rew_mean      | 153      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1340     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 14341    |\n",
      "|    total_timesteps  | 77988    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.198    |\n",
      "|    n_updates        | 19246    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 94.4     |\n",
      "|    ep_rew_mean      | 153      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1344     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 14387    |\n",
      "|    total_timesteps  | 78251    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.242    |\n",
      "|    n_updates        | 19312    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 95.5     |\n",
      "|    ep_rew_mean      | 156      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1348     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 14443    |\n",
      "|    total_timesteps  | 78616    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.45     |\n",
      "|    n_updates        | 19403    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 97.3     |\n",
      "|    ep_rew_mean      | 161      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1352     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 14524    |\n",
      "|    total_timesteps  | 79152    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.33     |\n",
      "|    n_updates        | 19537    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 99.3     |\n",
      "|    ep_rew_mean      | 167      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1356     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 14585    |\n",
      "|    total_timesteps  | 79558    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2        |\n",
      "|    n_updates        | 19639    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 99.1     |\n",
      "|    ep_rew_mean      | 167      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1360     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 14640    |\n",
      "|    total_timesteps  | 79901    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.263    |\n",
      "|    n_updates        | 19725    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 98.6     |\n",
      "|    ep_rew_mean      | 166      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1364     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 14696    |\n",
      "|    total_timesteps  | 80285    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.18     |\n",
      "|    n_updates        | 19821    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 97.3     |\n",
      "|    ep_rew_mean      | 162      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1368     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 14741    |\n",
      "|    total_timesteps  | 80581    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.205    |\n",
      "|    n_updates        | 19895    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 97.8     |\n",
      "|    ep_rew_mean      | 164      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1372     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 14803    |\n",
      "|    total_timesteps  | 80993    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.22     |\n",
      "|    n_updates        | 19998    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 97.1     |\n",
      "|    ep_rew_mean      | 162      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1376     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 14841    |\n",
      "|    total_timesteps  | 81235    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.289    |\n",
      "|    n_updates        | 20058    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 98.2     |\n",
      "|    ep_rew_mean      | 165      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1380     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 14892    |\n",
      "|    total_timesteps  | 81547    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.347    |\n",
      "|    n_updates        | 20136    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 96.3     |\n",
      "|    ep_rew_mean      | 160      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1384     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 14943    |\n",
      "|    total_timesteps  | 81834    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.53     |\n",
      "|    n_updates        | 20208    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 96.1     |\n",
      "|    ep_rew_mean      | 160      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1388     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15014    |\n",
      "|    total_timesteps  | 82283    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.71     |\n",
      "|    n_updates        | 20320    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 96.2     |\n",
      "|    ep_rew_mean      | 161      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1392     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15069    |\n",
      "|    total_timesteps  | 82619    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.39     |\n",
      "|    n_updates        | 20404    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 93.7     |\n",
      "|    ep_rew_mean      | 154      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1396     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15125    |\n",
      "|    total_timesteps  | 82981    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.757    |\n",
      "|    n_updates        | 20495    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 92.5     |\n",
      "|    ep_rew_mean      | 151      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1400     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15199    |\n",
      "|    total_timesteps  | 83462    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.56     |\n",
      "|    n_updates        | 20615    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 91       |\n",
      "|    ep_rew_mean      | 147      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1404     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15249    |\n",
      "|    total_timesteps  | 83786    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 6.22     |\n",
      "|    n_updates        | 20696    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.5     |\n",
      "|    ep_rew_mean      | 143      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1408     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15285    |\n",
      "|    total_timesteps  | 84007    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.359    |\n",
      "|    n_updates        | 20751    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 92.1     |\n",
      "|    ep_rew_mean      | 150      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1412     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15356    |\n",
      "|    total_timesteps  | 84459    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.22     |\n",
      "|    n_updates        | 20864    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.9     |\n",
      "|    ep_rew_mean      | 144      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1416     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15410    |\n",
      "|    total_timesteps  | 84762    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.464    |\n",
      "|    n_updates        | 20940    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.8     |\n",
      "|    ep_rew_mean      | 144      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1420     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15475    |\n",
      "|    total_timesteps  | 85161    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.586    |\n",
      "|    n_updates        | 21040    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.5     |\n",
      "|    ep_rew_mean      | 143      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1424     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15511    |\n",
      "|    total_timesteps  | 85385    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.611    |\n",
      "|    n_updates        | 21096    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87.6     |\n",
      "|    ep_rew_mean      | 138      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1428     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15558    |\n",
      "|    total_timesteps  | 85673    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.177    |\n",
      "|    n_updates        | 21168    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86.3     |\n",
      "|    ep_rew_mean      | 134      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1432     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15601    |\n",
      "|    total_timesteps  | 85902    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.54     |\n",
      "|    n_updates        | 21225    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 88.2     |\n",
      "|    ep_rew_mean      | 140      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1436     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15675    |\n",
      "|    total_timesteps  | 86409    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.227    |\n",
      "|    n_updates        | 21352    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 88.5     |\n",
      "|    ep_rew_mean      | 140      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1440     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15739    |\n",
      "|    total_timesteps  | 86833    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.262    |\n",
      "|    n_updates        | 21458    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.4     |\n",
      "|    ep_rew_mean      | 143      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1444     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15797    |\n",
      "|    total_timesteps  | 87192    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.7      |\n",
      "|    n_updates        | 21547    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 88.9     |\n",
      "|    ep_rew_mean      | 142      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1448     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15845    |\n",
      "|    total_timesteps  | 87502    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0877   |\n",
      "|    n_updates        | 21625    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86       |\n",
      "|    ep_rew_mean      | 134      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1452     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15886    |\n",
      "|    total_timesteps  | 87754    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.24     |\n",
      "|    n_updates        | 21688    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85       |\n",
      "|    ep_rew_mean      | 131      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1456     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15934    |\n",
      "|    total_timesteps  | 88061    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.58     |\n",
      "|    n_updates        | 21765    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.1     |\n",
      "|    ep_rew_mean      | 131      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1460     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 15995    |\n",
      "|    total_timesteps  | 88413    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.306    |\n",
      "|    n_updates        | 21853    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86.7     |\n",
      "|    ep_rew_mean      | 135      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1464     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 16079    |\n",
      "|    total_timesteps  | 88957    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.146    |\n",
      "|    n_updates        | 21989    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86.9     |\n",
      "|    ep_rew_mean      | 136      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1468     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 16127    |\n",
      "|    total_timesteps  | 89274    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.279    |\n",
      "|    n_updates        | 22068    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.4     |\n",
      "|    ep_rew_mean      | 132      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1472     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 16170    |\n",
      "|    total_timesteps  | 89531    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.47     |\n",
      "|    n_updates        | 22132    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.4     |\n",
      "|    ep_rew_mean      | 132      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1476     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 16207    |\n",
      "|    total_timesteps  | 89778    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.087    |\n",
      "|    n_updates        | 22194    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.5     |\n",
      "|    ep_rew_mean      | 132      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1480     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 16260    |\n",
      "|    total_timesteps  | 90101    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.248    |\n",
      "|    n_updates        | 22275    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86.6     |\n",
      "|    ep_rew_mean      | 135      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1484     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 16332    |\n",
      "|    total_timesteps  | 90493    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.66     |\n",
      "|    n_updates        | 22373    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.1     |\n",
      "|    ep_rew_mean      | 131      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1488     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 16388    |\n",
      "|    total_timesteps  | 90794    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.97     |\n",
      "|    n_updates        | 22448    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 84.4     |\n",
      "|    ep_rew_mean      | 128      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1492     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 16436    |\n",
      "|    total_timesteps  | 91057    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.45     |\n",
      "|    n_updates        | 22514    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 84.3     |\n",
      "|    ep_rew_mean      | 128      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1496     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 16495    |\n",
      "|    total_timesteps  | 91414    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.153    |\n",
      "|    n_updates        | 22603    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 83.5     |\n",
      "|    ep_rew_mean      | 126      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1500     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 16562    |\n",
      "|    total_timesteps  | 91808    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.86     |\n",
      "|    n_updates        | 22701    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 81.7     |\n",
      "|    ep_rew_mean      | 121      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1504     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 16593    |\n",
      "|    total_timesteps  | 91959    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.366    |\n",
      "|    n_updates        | 22739    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 83.1     |\n",
      "|    ep_rew_mean      | 123      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1508     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 16661    |\n",
      "|    total_timesteps  | 92313    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.358    |\n",
      "|    n_updates        | 22828    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 83.4     |\n",
      "|    ep_rew_mean      | 125      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1512     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 16735    |\n",
      "|    total_timesteps  | 92802    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.18     |\n",
      "|    n_updates        | 22950    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 83.4     |\n",
      "|    ep_rew_mean      | 124      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1516     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 16791    |\n",
      "|    total_timesteps  | 93104    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.91     |\n",
      "|    n_updates        | 23025    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 84.3     |\n",
      "|    ep_rew_mean      | 127      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1520     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 16875    |\n",
      "|    total_timesteps  | 93592    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.131    |\n",
      "|    n_updates        | 23147    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 84.8     |\n",
      "|    ep_rew_mean      | 127      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1524     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 16926    |\n",
      "|    total_timesteps  | 93861    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.32     |\n",
      "|    n_updates        | 23215    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.5     |\n",
      "|    ep_rew_mean      | 129      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1528     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 16994    |\n",
      "|    total_timesteps  | 94228    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.281    |\n",
      "|    n_updates        | 23306    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.3     |\n",
      "|    ep_rew_mean      | 128      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1532     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 17041    |\n",
      "|    total_timesteps  | 94429    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.511    |\n",
      "|    n_updates        | 23357    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 84       |\n",
      "|    ep_rew_mean      | 123      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1536     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 17114    |\n",
      "|    total_timesteps  | 94809    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.22     |\n",
      "|    n_updates        | 23452    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 82.8     |\n",
      "|    ep_rew_mean      | 119      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1540     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 17173    |\n",
      "|    total_timesteps  | 95110    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.15     |\n",
      "|    n_updates        | 23527    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 83.6     |\n",
      "|    ep_rew_mean      | 121      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1544     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 17246    |\n",
      "|    total_timesteps  | 95549    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.341    |\n",
      "|    n_updates        | 23637    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 84.3     |\n",
      "|    ep_rew_mean      | 123      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1548     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 17317    |\n",
      "|    total_timesteps  | 95936    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.93     |\n",
      "|    n_updates        | 23733    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87.1     |\n",
      "|    ep_rew_mean      | 129      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1552     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 17410    |\n",
      "|    total_timesteps  | 96461    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.776    |\n",
      "|    n_updates        | 23865    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86.1     |\n",
      "|    ep_rew_mean      | 127      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1556     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 17449    |\n",
      "|    total_timesteps  | 96674    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.23     |\n",
      "|    n_updates        | 23918    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.9     |\n",
      "|    ep_rew_mean      | 125      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1560     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 17520    |\n",
      "|    total_timesteps  | 97002    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.35     |\n",
      "|    n_updates        | 24000    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 84.3     |\n",
      "|    ep_rew_mean      | 120      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1564     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 17589    |\n",
      "|    total_timesteps  | 97389    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.141    |\n",
      "|    n_updates        | 24097    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 83.8     |\n",
      "|    ep_rew_mean      | 118      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1568     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 17639    |\n",
      "|    total_timesteps  | 97650    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.263    |\n",
      "|    n_updates        | 24162    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.7     |\n",
      "|    ep_rew_mean      | 123      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1572     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 17715    |\n",
      "|    total_timesteps  | 98102    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.251    |\n",
      "|    n_updates        | 24275    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87       |\n",
      "|    ep_rew_mean      | 126      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1576     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 17787    |\n",
      "|    total_timesteps  | 98473    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.38     |\n",
      "|    n_updates        | 24368    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89       |\n",
      "|    ep_rew_mean      | 131      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1580     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 17876    |\n",
      "|    total_timesteps  | 99005    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.137    |\n",
      "|    n_updates        | 24501    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.9     |\n",
      "|    ep_rew_mean      | 134      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1584     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 17954    |\n",
      "|    total_timesteps  | 99487    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.13     |\n",
      "|    n_updates        | 24621    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 90.7     |\n",
      "|    ep_rew_mean      | 136      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1588     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 18016    |\n",
      "|    total_timesteps  | 99862    |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.217    |\n",
      "|    n_updates        | 24715    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 93.7     |\n",
      "|    ep_rew_mean      | 145      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1592     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 18098    |\n",
      "|    total_timesteps  | 100426   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.266    |\n",
      "|    n_updates        | 24856    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 94.8     |\n",
      "|    ep_rew_mean      | 148      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1596     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 18171    |\n",
      "|    total_timesteps  | 100897   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.09     |\n",
      "|    n_updates        | 24974    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 93.7     |\n",
      "|    ep_rew_mean      | 145      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1600     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 18218    |\n",
      "|    total_timesteps  | 101177   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.06     |\n",
      "|    n_updates        | 25044    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 95.9     |\n",
      "|    ep_rew_mean      | 151      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1604     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 18273    |\n",
      "|    total_timesteps  | 101550   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.348    |\n",
      "|    n_updates        | 25137    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 95.1     |\n",
      "|    ep_rew_mean      | 150      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1608     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 18318    |\n",
      "|    total_timesteps  | 101822   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.81     |\n",
      "|    n_updates        | 25205    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 96.1     |\n",
      "|    ep_rew_mean      | 153      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1612     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 18404    |\n",
      "|    total_timesteps  | 102410   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.479    |\n",
      "|    n_updates        | 25352    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 95.2     |\n",
      "|    ep_rew_mean      | 151      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1616     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 18438    |\n",
      "|    total_timesteps  | 102627   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.01     |\n",
      "|    n_updates        | 25406    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 93.5     |\n",
      "|    ep_rew_mean      | 147      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1620     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 18487    |\n",
      "|    total_timesteps  | 102937   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.37     |\n",
      "|    n_updates        | 25484    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 92.9     |\n",
      "|    ep_rew_mean      | 145      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1624     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 18526    |\n",
      "|    total_timesteps  | 103148   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.361    |\n",
      "|    n_updates        | 25536    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 91.3     |\n",
      "|    ep_rew_mean      | 141      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1628     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 18566    |\n",
      "|    total_timesteps  | 103358   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.955    |\n",
      "|    n_updates        | 25589    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 93.9     |\n",
      "|    ep_rew_mean      | 149      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1632     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 18646    |\n",
      "|    total_timesteps  | 103820   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.357    |\n",
      "|    n_updates        | 25704    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 93.4     |\n",
      "|    ep_rew_mean      | 148      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1636     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 18702    |\n",
      "|    total_timesteps  | 104149   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.748    |\n",
      "|    n_updates        | 25787    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 94.8     |\n",
      "|    ep_rew_mean      | 152      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1640     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 18782    |\n",
      "|    total_timesteps  | 104589   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.647    |\n",
      "|    n_updates        | 25897    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 92.6     |\n",
      "|    ep_rew_mean      | 146      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1644     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 18812    |\n",
      "|    total_timesteps  | 104808   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.251    |\n",
      "|    n_updates        | 25951    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 93.2     |\n",
      "|    ep_rew_mean      | 148      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1648     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 18882    |\n",
      "|    total_timesteps  | 105257   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 7.55     |\n",
      "|    n_updates        | 26064    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 92       |\n",
      "|    ep_rew_mean      | 146      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1652     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 18950    |\n",
      "|    total_timesteps  | 105665   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.853    |\n",
      "|    n_updates        | 26166    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 93.2     |\n",
      "|    ep_rew_mean      | 149      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1656     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19001    |\n",
      "|    total_timesteps  | 105993   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.456    |\n",
      "|    n_updates        | 26248    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 96.9     |\n",
      "|    ep_rew_mean      | 160      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1660     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19104    |\n",
      "|    total_timesteps  | 106688   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.102    |\n",
      "|    n_updates        | 26421    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 95.9     |\n",
      "|    ep_rew_mean      | 158      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1664     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19149    |\n",
      "|    total_timesteps  | 106981   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.499    |\n",
      "|    n_updates        | 26495    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 97.2     |\n",
      "|    ep_rew_mean      | 162      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1668     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19215    |\n",
      "|    total_timesteps  | 107366   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.448    |\n",
      "|    n_updates        | 26591    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 94.7     |\n",
      "|    ep_rew_mean      | 156      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1672     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19249    |\n",
      "|    total_timesteps  | 107575   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.253    |\n",
      "|    n_updates        | 26643    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 94       |\n",
      "|    ep_rew_mean      | 155      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1676     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19294    |\n",
      "|    total_timesteps  | 107872   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.474    |\n",
      "|    n_updates        | 26717    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 92.1     |\n",
      "|    ep_rew_mean      | 150      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1680     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19347    |\n",
      "|    total_timesteps  | 108218   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.365    |\n",
      "|    n_updates        | 26804    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 90.9     |\n",
      "|    ep_rew_mean      | 147      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1684     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19405    |\n",
      "|    total_timesteps  | 108575   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.161    |\n",
      "|    n_updates        | 26893    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.7     |\n",
      "|    ep_rew_mean      | 144      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1688     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19451    |\n",
      "|    total_timesteps  | 108836   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.1      |\n",
      "|    n_updates        | 26958    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 88.1     |\n",
      "|    ep_rew_mean      | 139      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1692     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19516    |\n",
      "|    total_timesteps  | 109240   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.7      |\n",
      "|    n_updates        | 27059    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86.8     |\n",
      "|    ep_rew_mean      | 135      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1696     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19568    |\n",
      "|    total_timesteps  | 109577   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.591    |\n",
      "|    n_updates        | 27144    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86       |\n",
      "|    ep_rew_mean      | 133      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1700     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19603    |\n",
      "|    total_timesteps  | 109779   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.64     |\n",
      "|    n_updates        | 27194    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.7     |\n",
      "|    ep_rew_mean      | 132      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1704     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19656    |\n",
      "|    total_timesteps  | 110122   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.469    |\n",
      "|    n_updates        | 27280    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86.6     |\n",
      "|    ep_rew_mean      | 135      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1708     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19711    |\n",
      "|    total_timesteps  | 110486   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.542    |\n",
      "|    n_updates        | 27371    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 83.6     |\n",
      "|    ep_rew_mean      | 127      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1712     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19761    |\n",
      "|    total_timesteps  | 110767   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.09     |\n",
      "|    n_updates        | 27441    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 84.2     |\n",
      "|    ep_rew_mean      | 128      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1716     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19804    |\n",
      "|    total_timesteps  | 111046   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.178    |\n",
      "|    n_updates        | 27511    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.3     |\n",
      "|    ep_rew_mean      | 131      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1720     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19871    |\n",
      "|    total_timesteps  | 111466   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.296    |\n",
      "|    n_updates        | 27616    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 88.6     |\n",
      "|    ep_rew_mean      | 140      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1724     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19951    |\n",
      "|    total_timesteps  | 112005   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.109    |\n",
      "|    n_updates        | 27751    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.5     |\n",
      "|    ep_rew_mean      | 143      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1728     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 19996    |\n",
      "|    total_timesteps  | 112309   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.18     |\n",
      "|    n_updates        | 27827    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87.7     |\n",
      "|    ep_rew_mean      | 139      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1732     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 20036    |\n",
      "|    total_timesteps  | 112589   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.464    |\n",
      "|    n_updates        | 27897    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87.8     |\n",
      "|    ep_rew_mean      | 140      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1736     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 20089    |\n",
      "|    total_timesteps  | 112929   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.34     |\n",
      "|    n_updates        | 27982    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 88.6     |\n",
      "|    ep_rew_mean      | 142      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1740     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 20166    |\n",
      "|    total_timesteps  | 113450   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.504    |\n",
      "|    n_updates        | 28112    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 88.7     |\n",
      "|    ep_rew_mean      | 142      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1744     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 20211    |\n",
      "|    total_timesteps  | 113674   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.4      |\n",
      "|    n_updates        | 28168    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.9     |\n",
      "|    ep_rew_mean      | 145      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1748     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 20298    |\n",
      "|    total_timesteps  | 114249   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.16     |\n",
      "|    n_updates        | 28312    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 90.2     |\n",
      "|    ep_rew_mean      | 146      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1752     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 20367    |\n",
      "|    total_timesteps  | 114680   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.505    |\n",
      "|    n_updates        | 28419    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 88.9     |\n",
      "|    ep_rew_mean      | 142      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1756     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 20399    |\n",
      "|    total_timesteps  | 114879   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.04     |\n",
      "|    n_updates        | 28469    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86.6     |\n",
      "|    ep_rew_mean      | 136      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1760     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 20472    |\n",
      "|    total_timesteps  | 115351   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.169    |\n",
      "|    n_updates        | 28587    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86.2     |\n",
      "|    ep_rew_mean      | 135      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1764     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 20510    |\n",
      "|    total_timesteps  | 115602   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.75     |\n",
      "|    n_updates        | 28650    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.7     |\n",
      "|    ep_rew_mean      | 134      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1768     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 20563    |\n",
      "|    total_timesteps  | 115937   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.4      |\n",
      "|    n_updates        | 28734    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87       |\n",
      "|    ep_rew_mean      | 137      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1772     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 20617    |\n",
      "|    total_timesteps  | 116273   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.66     |\n",
      "|    n_updates        | 28818    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87.4     |\n",
      "|    ep_rew_mean      | 138      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1776     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 20666    |\n",
      "|    total_timesteps  | 116608   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.26     |\n",
      "|    n_updates        | 28901    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86.4     |\n",
      "|    ep_rew_mean      | 135      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1780     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 20710    |\n",
      "|    total_timesteps  | 116857   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.79     |\n",
      "|    n_updates        | 28964    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.5     |\n",
      "|    ep_rew_mean      | 133      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1784     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 20753    |\n",
      "|    total_timesteps  | 117130   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.87     |\n",
      "|    n_updates        | 29032    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86       |\n",
      "|    ep_rew_mean      | 134      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1788     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 20807    |\n",
      "|    total_timesteps  | 117431   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.179    |\n",
      "|    n_updates        | 29107    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85       |\n",
      "|    ep_rew_mean      | 131      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1792     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 20860    |\n",
      "|    total_timesteps  | 117736   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.508    |\n",
      "|    n_updates        | 29183    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 83.7     |\n",
      "|    ep_rew_mean      | 128      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1796     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 20899    |\n",
      "|    total_timesteps  | 117950   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.314    |\n",
      "|    n_updates        | 29237    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87.1     |\n",
      "|    ep_rew_mean      | 137      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1800     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 20984    |\n",
      "|    total_timesteps  | 118488   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.5      |\n",
      "|    n_updates        | 29371    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 86.2     |\n",
      "|    ep_rew_mean      | 134      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1804     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21025    |\n",
      "|    total_timesteps  | 118746   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.49     |\n",
      "|    n_updates        | 29436    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 84.9     |\n",
      "|    ep_rew_mean      | 131      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1808     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21066    |\n",
      "|    total_timesteps  | 118977   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.53     |\n",
      "|    n_updates        | 29494    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.5     |\n",
      "|    ep_rew_mean      | 133      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1812     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21115    |\n",
      "|    total_timesteps  | 119322   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.341    |\n",
      "|    n_updates        | 29580    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 85.7     |\n",
      "|    ep_rew_mean      | 133      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1816     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21169    |\n",
      "|    total_timesteps  | 119617   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.14     |\n",
      "|    n_updates        | 29654    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 84.1     |\n",
      "|    ep_rew_mean      | 128      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1820     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21211    |\n",
      "|    total_timesteps  | 119873   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.131    |\n",
      "|    n_updates        | 29718    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 82.1     |\n",
      "|    ep_rew_mean      | 123      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1824     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21263    |\n",
      "|    total_timesteps  | 120218   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.77     |\n",
      "|    n_updates        | 29804    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 82.7     |\n",
      "|    ep_rew_mean      | 124      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1828     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21323    |\n",
      "|    total_timesteps  | 120577   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 5.12     |\n",
      "|    n_updates        | 29894    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 83.2     |\n",
      "|    ep_rew_mean      | 126      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1832     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21373    |\n",
      "|    total_timesteps  | 120910   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.803    |\n",
      "|    n_updates        | 29977    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 82.9     |\n",
      "|    ep_rew_mean      | 125      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1836     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21421    |\n",
      "|    total_timesteps  | 121217   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.35     |\n",
      "|    n_updates        | 30054    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 80.9     |\n",
      "|    ep_rew_mean      | 119      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1840     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21473    |\n",
      "|    total_timesteps  | 121541   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.189    |\n",
      "|    n_updates        | 30135    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 81.8     |\n",
      "|    ep_rew_mean      | 122      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1844     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21520    |\n",
      "|    total_timesteps  | 121850   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.193    |\n",
      "|    n_updates        | 30212    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 80.1     |\n",
      "|    ep_rew_mean      | 117      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1848     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21589    |\n",
      "|    total_timesteps  | 122259   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.329    |\n",
      "|    n_updates        | 30314    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 78.9     |\n",
      "|    ep_rew_mean      | 114      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1852     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21640    |\n",
      "|    total_timesteps  | 122574   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.13     |\n",
      "|    n_updates        | 30393    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 79.7     |\n",
      "|    ep_rew_mean      | 116      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1856     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21684    |\n",
      "|    total_timesteps  | 122851   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.118    |\n",
      "|    n_updates        | 30462    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 77.3     |\n",
      "|    ep_rew_mean      | 110      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1860     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21719    |\n",
      "|    total_timesteps  | 123080   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.271    |\n",
      "|    n_updates        | 30519    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 77.2     |\n",
      "|    ep_rew_mean      | 110      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1864     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21763    |\n",
      "|    total_timesteps  | 123325   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.27     |\n",
      "|    n_updates        | 30581    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 77.4     |\n",
      "|    ep_rew_mean      | 110      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1868     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21816    |\n",
      "|    total_timesteps  | 123676   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.48     |\n",
      "|    n_updates        | 30668    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 76.9     |\n",
      "|    ep_rew_mean      | 109      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1872     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21861    |\n",
      "|    total_timesteps  | 123961   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.65     |\n",
      "|    n_updates        | 30740    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 76.4     |\n",
      "|    ep_rew_mean      | 107      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1876     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21908    |\n",
      "|    total_timesteps  | 124248   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.28     |\n",
      "|    n_updates        | 30811    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 76.8     |\n",
      "|    ep_rew_mean      | 108      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1880     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 21959    |\n",
      "|    total_timesteps  | 124532   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.635    |\n",
      "|    n_updates        | 30882    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 78.3     |\n",
      "|    ep_rew_mean      | 112      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1884     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22037    |\n",
      "|    total_timesteps  | 124960   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.38     |\n",
      "|    n_updates        | 30989    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 78.6     |\n",
      "|    ep_rew_mean      | 112      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1888     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22100    |\n",
      "|    total_timesteps  | 125289   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.545    |\n",
      "|    n_updates        | 31072    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 80.1     |\n",
      "|    ep_rew_mean      | 117      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1892     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22170    |\n",
      "|    total_timesteps  | 125745   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.945    |\n",
      "|    n_updates        | 31186    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 80.6     |\n",
      "|    ep_rew_mean      | 118      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1896     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22213    |\n",
      "|    total_timesteps  | 126006   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.38     |\n",
      "|    n_updates        | 31251    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 77.8     |\n",
      "|    ep_rew_mean      | 111      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1900     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22262    |\n",
      "|    total_timesteps  | 126273   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.61     |\n",
      "|    n_updates        | 31318    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 78.9     |\n",
      "|    ep_rew_mean      | 114      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1904     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22319    |\n",
      "|    total_timesteps  | 126634   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.166    |\n",
      "|    n_updates        | 31408    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 78.8     |\n",
      "|    ep_rew_mean      | 113      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1908     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22356    |\n",
      "|    total_timesteps  | 126857   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.69     |\n",
      "|    n_updates        | 31464    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 78.6     |\n",
      "|    ep_rew_mean      | 113      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1912     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22406    |\n",
      "|    total_timesteps  | 127178   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.762    |\n",
      "|    n_updates        | 31544    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 80       |\n",
      "|    ep_rew_mean      | 117      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1916     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22474    |\n",
      "|    total_timesteps  | 127622   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.06     |\n",
      "|    n_updates        | 31655    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 80.7     |\n",
      "|    ep_rew_mean      | 118      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1920     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22531    |\n",
      "|    total_timesteps  | 127942   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.635    |\n",
      "|    n_updates        | 31735    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 79.9     |\n",
      "|    ep_rew_mean      | 116      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1924     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22575    |\n",
      "|    total_timesteps  | 128211   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.278    |\n",
      "|    n_updates        | 31802    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 79.3     |\n",
      "|    ep_rew_mean      | 115      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1928     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22619    |\n",
      "|    total_timesteps  | 128507   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.429    |\n",
      "|    n_updates        | 31876    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 79.1     |\n",
      "|    ep_rew_mean      | 114      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1932     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22665    |\n",
      "|    total_timesteps  | 128817   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.06     |\n",
      "|    n_updates        | 31954    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 78.8     |\n",
      "|    ep_rew_mean      | 113      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1936     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22709    |\n",
      "|    total_timesteps  | 129097   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 3.39     |\n",
      "|    n_updates        | 32024    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 78.4     |\n",
      "|    ep_rew_mean      | 113      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1940     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22750    |\n",
      "|    total_timesteps  | 129382   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.233    |\n",
      "|    n_updates        | 32095    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 77.6     |\n",
      "|    ep_rew_mean      | 111      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1944     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22783    |\n",
      "|    total_timesteps  | 129610   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0622   |\n",
      "|    n_updates        | 32152    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 77.2     |\n",
      "|    ep_rew_mean      | 110      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1948     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22840    |\n",
      "|    total_timesteps  | 129983   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.566    |\n",
      "|    n_updates        | 32245    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 77.9     |\n",
      "|    ep_rew_mean      | 112      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1952     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22899    |\n",
      "|    total_timesteps  | 130361   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.47     |\n",
      "|    n_updates        | 32340    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 78.4     |\n",
      "|    ep_rew_mean      | 113      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1956     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 22949    |\n",
      "|    total_timesteps  | 130690   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.26     |\n",
      "|    n_updates        | 32422    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 80.5     |\n",
      "|    ep_rew_mean      | 119      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1960     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 23015    |\n",
      "|    total_timesteps  | 131130   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.177    |\n",
      "|    n_updates        | 32532    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 82.4     |\n",
      "|    ep_rew_mean      | 124      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1964     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 23079    |\n",
      "|    total_timesteps  | 131563   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.466    |\n",
      "|    n_updates        | 32640    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 84.2     |\n",
      "|    ep_rew_mean      | 129      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1968     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 23166    |\n",
      "|    total_timesteps  | 132096   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.108    |\n",
      "|    n_updates        | 32773    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 83.1     |\n",
      "|    ep_rew_mean      | 126      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1972     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 23197    |\n",
      "|    total_timesteps  | 132271   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.59     |\n",
      "|    n_updates        | 32817    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 84.2     |\n",
      "|    ep_rew_mean      | 129      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1976     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 23253    |\n",
      "|    total_timesteps  | 132663   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.29     |\n",
      "|    n_updates        | 32915    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 83.9     |\n",
      "|    ep_rew_mean      | 128      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1980     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 23293    |\n",
      "|    total_timesteps  | 132918   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0726   |\n",
      "|    n_updates        | 32979    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 82.2     |\n",
      "|    ep_rew_mean      | 124      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1984     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 23334    |\n",
      "|    total_timesteps  | 133176   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.125    |\n",
      "|    n_updates        | 33043    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 83.2     |\n",
      "|    ep_rew_mean      | 127      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1988     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 23403    |\n",
      "|    total_timesteps  | 133612   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.937    |\n",
      "|    n_updates        | 33152    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 82.4     |\n",
      "|    ep_rew_mean      | 125      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1992     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 23460    |\n",
      "|    total_timesteps  | 133982   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.684    |\n",
      "|    n_updates        | 33245    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 82.2     |\n",
      "|    ep_rew_mean      | 124      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 1996     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 23503    |\n",
      "|    total_timesteps  | 134227   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.217    |\n",
      "|    n_updates        | 33306    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87.2     |\n",
      "|    ep_rew_mean      | 139      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2000     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 23610    |\n",
      "|    total_timesteps  | 134998   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.58     |\n",
      "|    n_updates        | 33499    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89       |\n",
      "|    ep_rew_mean      | 144      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2004     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 23693    |\n",
      "|    total_timesteps  | 135539   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.5      |\n",
      "|    n_updates        | 33634    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.7     |\n",
      "|    ep_rew_mean      | 146      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2008     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 23740    |\n",
      "|    total_timesteps  | 135831   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.11     |\n",
      "|    n_updates        | 33707    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.5     |\n",
      "|    ep_rew_mean      | 145      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2012     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 23792    |\n",
      "|    total_timesteps  | 136133   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 2.9      |\n",
      "|    n_updates        | 33783    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87.8     |\n",
      "|    ep_rew_mean      | 140      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2016     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 23833    |\n",
      "|    total_timesteps  | 136407   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.25     |\n",
      "|    n_updates        | 33851    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 88.4     |\n",
      "|    ep_rew_mean      | 142      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2020     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 23893    |\n",
      "|    total_timesteps  | 136781   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.122    |\n",
      "|    n_updates        | 33945    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.7     |\n",
      "|    ep_rew_mean      | 146      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2024     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 23954    |\n",
      "|    total_timesteps  | 137177   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.213    |\n",
      "|    n_updates        | 34044    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.8     |\n",
      "|    ep_rew_mean      | 146      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2028     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 24001    |\n",
      "|    total_timesteps  | 137492   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.199    |\n",
      "|    n_updates        | 34122    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.7     |\n",
      "|    ep_rew_mean      | 146      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2032     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 24045    |\n",
      "|    total_timesteps  | 137790   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.03     |\n",
      "|    n_updates        | 34197    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 91.1     |\n",
      "|    ep_rew_mean      | 150      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2036     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 24110    |\n",
      "|    total_timesteps  | 138204   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 4.28     |\n",
      "|    n_updates        | 34300    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 91.5     |\n",
      "|    ep_rew_mean      | 151      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2040     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 24163    |\n",
      "|    total_timesteps  | 138536   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.982    |\n",
      "|    n_updates        | 34383    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 91.5     |\n",
      "|    ep_rew_mean      | 150      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2044     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 24197    |\n",
      "|    total_timesteps  | 138761   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0973   |\n",
      "|    n_updates        | 34440    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 90.3     |\n",
      "|    ep_rew_mean      | 147      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2048     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 24237    |\n",
      "|    total_timesteps  | 139013   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.632    |\n",
      "|    n_updates        | 34503    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 88.7     |\n",
      "|    ep_rew_mean      | 143      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2052     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 24273    |\n",
      "|    total_timesteps  | 139235   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.192    |\n",
      "|    n_updates        | 34558    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 87.8     |\n",
      "|    ep_rew_mean      | 140      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2056     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 24313    |\n",
      "|    total_timesteps  | 139466   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.48     |\n",
      "|    n_updates        | 34616    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.3     |\n",
      "|    ep_rew_mean      | 144      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2060     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 24405    |\n",
      "|    total_timesteps  | 140060   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.345    |\n",
      "|    n_updates        | 34764    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 90.7     |\n",
      "|    ep_rew_mean      | 148      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2064     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 24491    |\n",
      "|    total_timesteps  | 140631   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.51     |\n",
      "|    n_updates        | 34907    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.2     |\n",
      "|    ep_rew_mean      | 144      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2068     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 24554    |\n",
      "|    total_timesteps  | 141013   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.262    |\n",
      "|    n_updates        | 35003    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 91.3     |\n",
      "|    ep_rew_mean      | 149      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2072     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 24623    |\n",
      "|    total_timesteps  | 141405   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.66     |\n",
      "|    n_updates        | 35101    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 89.3     |\n",
      "|    ep_rew_mean      | 144      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2076     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 24659    |\n",
      "|    total_timesteps  | 141598   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.472    |\n",
      "|    n_updates        | 35149    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 92.5     |\n",
      "|    ep_rew_mean      | 152      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2080     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 24749    |\n",
      "|    total_timesteps  | 142171   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.135    |\n",
      "|    n_updates        | 35292    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 93.3     |\n",
      "|    ep_rew_mean      | 154      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2084     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 24802    |\n",
      "|    total_timesteps  | 142506   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.141    |\n",
      "|    n_updates        | 35376    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 93.6     |\n",
      "|    ep_rew_mean      | 155      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2088     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 24873    |\n",
      "|    total_timesteps  | 142970   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.431    |\n",
      "|    n_updates        | 35492    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 96.9     |\n",
      "|    ep_rew_mean      | 164      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2092     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 24989    |\n",
      "|    total_timesteps  | 143673   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.162    |\n",
      "|    n_updates        | 35668    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 96.3     |\n",
      "|    ep_rew_mean      | 162      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2096     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 25023    |\n",
      "|    total_timesteps  | 143858   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.187    |\n",
      "|    n_updates        | 35714    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 94.1     |\n",
      "|    ep_rew_mean      | 156      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2100     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 25107    |\n",
      "|    total_timesteps  | 144409   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.218    |\n",
      "|    n_updates        | 35852    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 94.2     |\n",
      "|    ep_rew_mean      | 156      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2104     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 25193    |\n",
      "|    total_timesteps  | 144954   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 1.23     |\n",
      "|    n_updates        | 35988    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 94.4     |\n",
      "|    ep_rew_mean      | 156      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2108     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 25245    |\n",
      "|    total_timesteps  | 145269   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.0604   |\n",
      "|    n_updates        | 36067    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 95       |\n",
      "|    ep_rew_mean      | 158      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2112     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 25298    |\n",
      "|    total_timesteps  | 145637   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.601    |\n",
      "|    n_updates        | 36159    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 96.2     |\n",
      "|    ep_rew_mean      | 161      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2116     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 25364    |\n",
      "|    total_timesteps  | 146022   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.724    |\n",
      "|    n_updates        | 36255    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 97.3     |\n",
      "|    ep_rew_mean      | 164      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2120     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 25443    |\n",
      "|    total_timesteps  | 146511   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.205    |\n",
      "|    n_updates        | 36377    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 98       |\n",
      "|    ep_rew_mean      | 165      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2124     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 25518    |\n",
      "|    total_timesteps  | 146972   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.286    |\n",
      "|    n_updates        | 36492    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 99.6     |\n",
      "|    ep_rew_mean      | 169      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2128     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 25606    |\n",
      "|    total_timesteps  | 147453   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.259    |\n",
      "|    n_updates        | 36613    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 101      |\n",
      "|    ep_rew_mean      | 172      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2132     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 25679    |\n",
      "|    total_timesteps  | 147893   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.348    |\n",
      "|    n_updates        | 36723    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 99.1     |\n",
      "|    ep_rew_mean      | 167      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2136     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 25713    |\n",
      "|    total_timesteps  | 148117   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.87     |\n",
      "|    n_updates        | 36779    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 99.7     |\n",
      "|    ep_rew_mean      | 168      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2140     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 25781    |\n",
      "|    total_timesteps  | 148501   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.971    |\n",
      "|    n_updates        | 36875    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 102      |\n",
      "|    ep_rew_mean      | 173      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2144     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 25860    |\n",
      "|    total_timesteps  | 148924   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.135    |\n",
      "|    n_updates        | 36980    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 105      |\n",
      "|    ep_rew_mean      | 181      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2148     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 25945    |\n",
      "|    total_timesteps  | 149492   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 0.133    |\n",
      "|    n_updates        | 37122    |\n",
      "----------------------------------\n",
      "----------------------------------\n",
      "| rollout/            |          |\n",
      "|    ep_len_mean      | 106      |\n",
      "|    ep_rew_mean      | 185      |\n",
      "|    exploration_rate | 0.05     |\n",
      "| time/               |          |\n",
      "|    episodes         | 2152     |\n",
      "|    fps              | 5        |\n",
      "|    time_elapsed     | 26003    |\n",
      "|    total_timesteps  | 149857   |\n",
      "| train/              |          |\n",
      "|    learning_rate    | 0.0001   |\n",
      "|    loss             | 6        |\n",
      "|    n_updates        | 37214    |\n",
      "----------------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.dqn.dqn.DQN at 0x2b3a12d40>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.learn(total_timesteps=150000, callback=callback)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "e03f2eb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "best_model = DQN.load(os.path.join('train', 'best_model_150000'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2dc0b8b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "for episode in range(10):\n",
    "    obs, info = env.reset()\n",
    "    done = False\n",
    "    total_reward = 0\n",
    "    while not done:\n",
    "        action, _ =  best_model.predict(obs)\n",
    "        obs, reward, done, _, info = env.step(int(action))\n",
    "        total_reward += reward\n",
    "    print(f'Total reward for episode {episode + 1 } is {total_reward}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  },
  "vscode": {
   "interpreter": {
    "hash": "c6e4e9f98eb68ad3b7c296f83d20e6de614cb42e90992a65aa266555a3137d0d"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}