DQN Training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep Q-Network Training\n",
    "\n",
    "In this worksheet we are going to go over training a Reinforcement Learning (RL) agent trained on ATARI 2600 games. We will use the famous game of pong as a our training test bed.\n",
    "\n",
    "One can download the code required to train the DQN agent at https://sites.google.com/a/deepmind.com/dqn/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup\n",
    "We first start by setting up a function to update our lua path so that DQN training code can be loaded."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "--\n",
    "-- Most probably this function should have been part of statndard lua or Torch\n",
    "--\n",
    "function addpath(new_path)\n",
    "    package.path = package.path .. ';' .. new_path .. '/?.lua;' .. new_path .. '/?/init.lua'\n",
    "end\n",
    "--\n",
    "-- This is where the code lives on your machine.\n",
    "--\n",
    "addpath('/home/ubuntu/DQN/Human_Level_Control_through_Deep_Reinforcement_Learning/dqn/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  0 : train_agent.lua\n",
       "  1 : -framework\n",
       "  2 : alewrap\n",
       "  3 : -game_path\n",
       "  4 : /home/ubuntu/DQN/Human_Level_Control_through_Deep_Reinforcement_Learning/roms/\n",
       "  5 : -name\n",
       "  6 : DQN3_0_1_pong_FULL_Y\n",
       "  7 : -env\n",
       "  8 : pong\n",
       "  9 : -env_params\n",
       "  10 : useRGB=true\n",
       "  11 : -agent\n",
       "  12 : NeuralQLearner\n",
       "  13 : -agent_params\n",
       "  14 : lr=0.00025,ep=1,ep_end=0.1,ep_endt=replay_memory,discount=0.99,hist_len=4,learn_start=50000,replay_memory=1000000,update_freq=4,n_replay=1,network=\"convnet_atari3\",preproc=\"net_downsample_2x_full_y\",state_dim=7056,minibatch_size=32,rescale_r=1,ncols=1,bufferSize=512,valid_size=500,target_q=10000,clip_delta=1,min_reward=-1,max_reward=1\n",
       "  15 : -steps\n",
       "  16 : 50000000\n",
       "  17 : -eval_freq\n",
       "  18 : 250000\n",
       "  19 : -eval_steps\n",
       "  20 : 125000\n",
       "  21 : -prog_freq\n",
       "  22 : 10000\n",
       "  23 : -save_freq\n",
       "  24 : 125000\n",
       "  25 : -actrep\n",
       "  26 : 4\n",
       "  27 : -gpu\n",
       "  28 : 0\n",
       "  29 : -random_starts\n",
       "  30 : 30\n",
       "  31 : -pool_frms\n",
       "  32 : type=\"max\",size=2\n",
       "  33 : -seed\n",
       "  34 : 1\n",
       "  35 : -threads\n",
       "  36 : 4\n",
       "  -2 : -l\n",
       "  -1 : env\n",
       "  -3 : /home/ubuntu/torch/install/bin/luajit\n",
       "}\n"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "--\n",
    "-- The training arguments are stored in a file in the following file. We load them here and then explain one by one.\n",
    "-- In lua, when a process is run a global 'arg' table contains all the command line arguments, just like **argv.\n",
    "--\n",
    "arg = torch.load('/home/ubuntu/DQN/Human_Level_Control_through_Deep_Reinforcement_Learning/dqn/pong_arguments.t7')\n",
    "print(arg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "--\n",
    "-- Load initialization functions\n",
    "--\n",
    "require \"initenv\";"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "--\n",
    "-- We start by defining the valid command line arguments. Each argument is defined by its flag, followed by its\n",
    "-- default value and finally a help string.\n",
    "--\n",
    "cmd = torch.CmdLine()\n",
    "cmd:text()\n",
    "cmd:text('Train Agent in Environment:')\n",
    "cmd:text()\n",
    "cmd:text('Options:')\n",
    "\n",
    "cmd:option('-framework', '', 'name of training framework')\n",
    "cmd:option('-env', '', 'name of environment to use')\n",
    "cmd:option('-game_path', '', 'path to environment file (ROM)')\n",
    "cmd:option('-env_params', '', 'string of environment parameters')\n",
    "cmd:option('-pool_frms', '',\n",
    "           'string of frame pooling parameters (e.g.: size=2,type=\"max\")')\n",
    "cmd:option('-actrep', 1, 'how many times to repeat action')\n",
    "cmd:option('-random_starts', 0, 'play action 0 between 1 and random_starts ' ..\n",
    "           'number of times at the start of each training episode')\n",
    "cmd:option('-name', '', 'filename used for saving network and training history')\n",
    "cmd:option('-network', '', 'reload pretrained network')\n",
    "cmd:option('-agent', '', 'name of agent file to use')\n",
    "cmd:option('-agent_params', '', 'string of agent parameters')\n",
    "cmd:option('-seed', 1, 'fixed input seed for repeatable experiments')\n",
    "cmd:option('-saveNetworkParams', false,\n",
    "           'saves the agent network in a separate file')\n",
    "cmd:option('-prog_freq', 5*10^3, 'frequency of progress output')\n",
    "cmd:option('-save_freq', 5*10^4, 'the model is saved every save_freq steps')\n",
    "cmd:option('-eval_freq', 10^4, 'frequency of greedy evaluation')\n",
    "cmd:option('-save_versions', 0, '')\n",
    "\n",
    "cmd:option('-steps', 10^5, 'number of training steps to perform')\n",
    "cmd:option('-eval_steps', 10^5, 'number of evaluation steps')\n",
    "cmd:option('-verbose', 2,\n",
    "           'the higher the level, the more information is printed to screen')\n",
    "cmd:option('-threads', 1, 'number of BLAS threads')\n",
    "cmd:option('-gpu', -1, 'gpu flag')\n",
    "\n",
    "cmd:text()\n",
    "\n",
    "-- Here we parse the table arg to load all command line arguments into opt\n",
    "opt = cmd:parse(arg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  eval_steps : 125000\n",
       "  seed : 1\n",
       "  name : DQN3_0_1_pong_FULL_Y\n",
       "  verbose : 2\n",
       "  network : \n",
       "  pool_frms : type=\"max\",size=2\n",
       "  saveNetworkParams : false\n",
       "  gpu : 0\n",
       "  eval_freq : 250000\n",
       "  env_params : useRGB=true\n",
       "  game_path : /home/ubuntu/DQN/Human_Level_Control_through_Deep_Reinforcement_Learning/roms/\n",
       "  prog_freq : 10000\n",
       "  agent_params : lr=0.00025,ep=1,ep_end=0.1,ep_endt=replay_memory,discount=0.99,hist_len=4,learn_start=50000,replay_memory=1000000,update_freq=4,n_replay=1,network=\"convnet_atari3\",preproc=\"net_downsample_2x_full_y\",state_dim=7056,minibatch_size=32,rescale_r=1,ncols=1,bufferSize=512,valid_size=500,target_q=10000,clip_delta=1,min_reward=-1,max_reward=1\n",
       "  env : pong\n",
       "  framework : alewrap\n",
       "  agent : NeuralQLearner\n",
       "  threads : 4\n",
       "  actrep : 4\n",
       "  random_starts : 30\n",
       "  save_versions : 0\n",
       "  save_freq : 125000\n",
       "  steps : 50000000\n",
       "}\n",
       "\n"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "--\n",
    "-- Observe the parameters that are required to specify the training.\n",
    "--\n",
    "print(opt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Torch Threads:\t4\t\n"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Using GPU device id:\t0\t\n",
       "Torch Seed:\t1\t\n"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "CUTorch Seed:\t1791095845\t\n"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "\n",
       "Playing:\tpong\t\n"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Creating Agent Network from convnet_atari3\t\n"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "nn.Sequential {\n",
       "  [input -> (1) -> (2) -> (3) -> (4) -> (5) -> (6) -> (7) -> (8) -> (9) -> (10) -> (11) -> output]\n",
       "  (1): nn.Reshape(4x84x84)\n",
       "  (2): nn.SpatialConvolution(in: 4, out: 32, kW: 8, kH: 8, dW: 4, dH: 4, padding: 1)\n",
       "  (3): nn.Rectifier\n",
       "  (4): nn.SpatialConvolution(in: 32, out: 64, kW: 4, kH: 4, dW: 2, dH: 2)\n",
       "  (5): nn.Rectifier\n",
       "  (6): nn.SpatialConvolution(in: 64, out: 64, kW: 3, kH: 3)\n",
       "  (7): nn.Rectifier\n",
       "  (8): nn.Reshape(3136)\n",
       "  (9): nn.Linear(3136 -> 512)\n",
       "  (10): nn.Rectifier\n",
       "  (11): nn.Linear(512 -> 3)\n",
       "}\n",
       "{\n",
       "  gradInput : CudaTensor - empty\n",
       "  modules : \n",
       "    {\n",
       "      1 : \n",
       "        nn.Reshape(4x84x84)\n",
       "        {\n",
       "          nelement : 28224\n",
       "          _input : CudaTensor - empty\n",
       "          output : CudaTensor - size: 1x4x84x84\n",
       "          gradInput : CudaTensor - empty\n",
       "          size : LongStorage - size: 3\n",
       "          _gradOutput : CudaTensor - empty\n",
       "          batchsize : LongStorage - size: 4\n",
       "        }\n",
       "      2 : \n",
       "    "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "    nn.SpatialConvolution(in: 4, out: 32, kW: 8, kH: 8, dW: 4, dH: 4, padding: 1)\n",
       "        {\n",
       "          dH : 4\n",
       "          dW : 4\n",
       "          nOutputPlane : 32\n",
       "          output : CudaTensor - size: 1x32x20x20\n",
       "          gradInput : CudaTensor - empty\n",
       "          finput : CudaTensor - empty\n",
       "          fgradInput : CudaTensor - empty\n",
       "          gradBias : CudaTensor - size: 32\n",
       "          weight : CudaTensor - size: 32x4x8x8\n",
       "          bias : CudaTensor - size: 32\n",
       "          gradWeight : CudaTensor - size: 32x4x8x8\n",
       "          padding : 1\n",
       "          nInputPlane : 4\n",
       "          kW : 8\n",
       "          kH : 8\n",
       "        }\n",
       "      3 : \n",
       "        nn.Rectifier\n",
       "        {\n",
       "          gradInput : CudaTensor - empty\n",
       "          output : CudaTensor - size: 1x32x20x20\n",
       "        }\n",
       "      4 : \n",
       "        nn.SpatialConvolution(in: 32, out: 64, kW: 4, kH: 4, dW: 2, dH: 2)\n",
       "        {\n",
       "          dH : 2\n",
       "          dW : 2\n",
       "          nOutputPlane : 64\n",
       "          output : CudaTensor - size: 1x64x9x9\n",
       "          gradInput : CudaTensor - empty\n",
       "          finput : CudaTensor - empty\n",
       "          fgradInput : CudaTensor - empty\n",
       "          gradBias : CudaTensor - size: 64\n",
       "          weight : CudaTensor - size: 64x32x4x4\n",
       "          bias : CudaTensor - size: 64\n",
       "          gradWeight : CudaTensor - size: 64x32x4x4\n",
       "          padding : 0\n",
       "          nInputPlane : 32\n",
       "          kW : 4\n",
       "          kH : 4\n",
       "        }\n",
       "      5 : \n",
       "        nn.Rectifier\n",
       "        {\n",
       "          gradInput : CudaTensor - empty\n",
       "          output : CudaTensor - size: 1x64x9x9\n",
       "        }\n",
       "      6 : \n",
       "        nn.SpatialConvolution(in: 64, out: 64, kW: 3, kH: 3)\n",
       "        {\n",
       "          dH : 1\n",
       "          dW : 1\n",
       "          nOutputPlane : 64\n",
       "          output : CudaTensor - size: 1x64x7x7\n",
       "          gradInput : CudaTensor - empty\n",
       " "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "         finput : CudaTensor - empty\n",
       "          fgradInput : CudaTensor - empty\n",
       "          gradBias : CudaTensor - size: 64\n",
       "          weight : CudaTensor - size: 64x64x3x3\n",
       "          bias : CudaTensor - size: 64\n",
       "          gradWeight : CudaTensor - size: 64x64x3x3\n",
       "          padding : 0\n",
       "          nInputPlane : 64\n",
       "          kW : 3\n",
       "          kH : 3\n",
       "        }\n",
       "      7 : \n",
       "        nn.Rectifier\n",
       "        {\n",
       "          gradInput : CudaTensor - empty\n",
       "          output : CudaTensor - size: 1x64x7x7\n",
       "        }\n",
       "      8 : \n",
       "        nn.Reshape(3136)\n",
       "        {\n",
       "          nelement : 3136\n",
       "          _input : CudaTensor - empty\n",
       "          output : CudaTensor - empty\n",
       "          gradInput : CudaTensor - empty\n",
       "          size : LongStorage - size: 1\n",
       "          _gradOutput : CudaTensor - empty\n",
       "          batchsize : LongStorage - size: 2\n",
       "        }\n",
       "      9 : \n",
       "        nn.Linear(3136 -> 512)\n",
       "        {\n",
       "          gradBias : CudaTensor - size: 512\n",
       "          weight : CudaTensor - size: 512x3136\n",
       "          bias : CudaTensor - size: 512\n",
       "          gradInput : CudaTensor - empty\n",
       "          gradWeight : CudaTensor - size: 512x3136\n",
       "          output : CudaTensor - empty\n",
       "        }\n",
       "      10 : \n",
       "        nn.Rectifier\n",
       "        {\n",
       "          gradInput : CudaTensor - empty\n",
       "          output : CudaTensor - empty\n",
       "        }\n",
       "      11 : \n",
       "        nn.Linear(512 -> 3)\n",
       "        {\n",
       "          gradBias : CudaTensor - size: 3\n",
       "          weight : CudaTensor - size: 3x512\n",
       "          bias : CudaTensor - size: 3\n",
       "          gradInput : CudaTensor - empty\n",
       "          gradWeight : CudaTensor - size: 3x512\n",
       "          output : CudaTensor - empty\n",
       "        }\n",
       "    }\n",
       "  output : CudaTensor - empty\n",
       "}\n",
       "Convolutional layers flattened output size:\t3136\t\n"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Set up Torch using these options:\t\n",
       "eval_steps\t125000\t\n",
       "seed\t1\t\n",
       "name\tDQN3_0_1_pong_FULL_Y\t\n",
       "verbose\t2\t\n",
       "network\t\t\n",
       "pool_frms\t{\n",
       "  type : max\n",
       "  size : 2\n",
       "}\n",
       "saveNetworkParams\tfalse\t\n",
       "gpu\t1\t\n",
       "eval_freq\t250000\t\n",
       "tensorType\ttorch.FloatTensor\t\n",
       "env_params\t{\n",
       "  useRGB : true\n",
       "}\n",
       "steps\t50000000\t\n",
       "prog_freq\t10000\t\n",
       "agent_params\t{\n",
       "  target_q : 10000\n",
       "  ncols : 1\n",
       "  replay_memory : 1000000\n",
       "  min_reward : -1\n",
       "  discount : 0.99\n",
       "  bufferSize : 512\n",
       "  hist_len : 4\n",
       "  ep : 1\n",
       "  network : convnet_atari3\n",
       "  max_reward : 1\n",
       "  gpu : 0\n",
       "  n_replay : 1\n",
       "  verbose : 2\n",
       "  ep_end : 0.1\n",
       "  lr : 0.00025\n",
       "  preproc : net_downsample_2x_full_y\n",
       "  valid_size : 500\n",
       "  update_freq : 4\n",
       "  minibatch_size : 32\n",
       "  rescale_r : 1\n",
       "  clip_delta : 1\n",
       "  state_dim : 7056\n",
       "  learn_start : 50000\n",
       "}\n",
       "save_versions\t0\t\n",
       "framework\talewrap\t\n",
       "agent\tNeuralQLearner\t\n",
       "threads\t4\t\n",
       "actrep\t4\t\n",
       "random_starts\t30\t\n",
       "game_path\t/home/ubuntu/DQN/Human_Level_Control_through_Deep_Reinforcement_Learning/roms/\t\n",
       "save_freq\t125000\t\n",
       "env\tpong\t\n"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "-- We do not have too much ram, so make sure to deallocate the current agent.\n",
    "if agent then\n",
    "    agent = nil\n",
    "    collectgarbage()\n",
    "    collectgarbage()\n",
    "end\n",
    "--\n",
    "-- Main setup function\n",
    "-- game : the game environment\n",
    "-- game_actions : valid actions that can be used in this game\n",
    "-- agent : the RL agent that we will train\n",
    "--\n",
    "-- if you don't like too much debug info in ipython\n",
    "--opt.verbose = 0\n",
    "\n",
    "-- run setup to load agent and game\n",
    "game_env, game_actions, agent, opt = setup(opt);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  eval_steps : 125000\n",
       "  seed : 1\n",
       "  name : DQN3_0_1_pong_FULL_Y\n",
       "  verbose : 2\n",
       "  network : \n",
       "  pool_frms : \n",
       "    {\n",
       "      type : max\n",
       "      size : 2\n",
       "    }\n",
       "  saveNetworkParams : false\n",
       "  gpu : 1\n",
       "  eval_freq : 250000\n",
       "  tensorType : torch.FloatTensor\n",
       "  env_params : \n",
       "    {\n",
       "      useRGB : true\n",
       "    }\n",
       "  steps : 50000000\n",
       "  prog_freq : 10000\n",
       "  agent_params : \n",
       "    {\n",
       "      target_q : 10000\n",
       "      ncols : 1\n",
       "      replay_memory : 1000000\n",
       "      min_reward : -1\n",
       "      discount : 0.99\n",
       "      bufferSize : 512\n",
       "      hist_len : 4\n",
       "      ep : 1\n",
       "      network : convnet_atari3\n",
       "      max_reward : 1\n",
       "      gpu : 0\n",
       "      n_replay : 1\n",
       "      verbose : 2\n",
       "      ep_end : 0.1\n",
       "      lr : 0.00025\n",
       "      preproc : net_downsample_2x_full_y\n",
       "      valid_size : 500\n",
       "      update_freq : 4\n",
       "      minibatch_size : 32\n",
       "      rescale_r : 1\n",
       "      clip_delta : 1\n",
       "      state_dim : 7056\n",
       "      learn_start : 50000\n",
       "    }\n",
       "  save_versions : 0\n",
       "  framework : alewrap\n",
       "  agent : NeuralQLearner\n",
       "  threads : 4\n",
       "  actrep : 4\n",
       "  random_starts : 30\n",
       "  game_path : /home/ubuntu/DQN/Human_Level_Control_through_Deep_Reinforcement_Learning/roms/\n",
       "  save_freq : 125000\n",
       "  env : pong\n",
       "}\n"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(opt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\n",
       "  1 : 0\n",
       "  2 : 3\n",
       "  3 : 4\n",
       "}\n"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA8wAAAT4CAIAAABCBtwMAAAgAElEQVR4nO3cQWpbW6KG0aOHRnA9GXeMB2CuwXPQLNz2KOI5GAQZQEjHk9EY9BpV5OVVJbGv/R0d6ey1Woa40IafS33Z7GhzPB4nAACgs52mabPZ/PLPvvz912kPwyx2+8Nk5bWz8gisPAIrj8DKI9jtD9vptHPe3d68+Ttfv30/wUlO4/Xx4c3fuX56OcFJrDwfK/+Oledg5flY+XesPAcrz+dMVv6fuT8AAABGI7IBACAmsgEAILZd8LN/fv3zntdCl+7n1z/veS20DlYegZVHYOURWHkEVj4ZN9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAENsu+Nl3tzcLfvrpvT4+LH2EBVh5BFYegZVHYOURWPlk3GQDAEBMZAMAQExkAwBAbHM8Hp/vr5Y+BjPa7Q/TNFl53aw8AiuPwMojsPIIdvuDm2wAAIiJbAAAiG2Ox+PSZwAAgFVxkw0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAACx7TRNz/dXSx+DGe32h8nKa2flEVh5BFYegZVHsNsf3GQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBsu/QBeJfXx4cfP18/vSx4EuZj5RFYeQRWHoGVR/DJld1kAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQGy79AF4l+unl6WPwOysPAIrj8DKI7DyCD65sptsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiG2Ox+PSZwAAgFVxkw0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAse00TZvN5pd/9uXvv057GGax2x8mK6+dlUdg5RFYeQRWHsFuf9hOp53z7vbmzd/5+u37CU5yGq+PD2/+zvXTywlOYuX5WPl3rDwHK8/Hyr9j5TlYeT5nsrLnIgAAEBPZAAAQE9kAABDbLvjZP7/+ec9roUv38+uf97wWWgcrj8DKI7DyCKw8AiufjJtsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIhtF/zsu9ubBT/99F4fH5Y+wgKsPAIrj8DKI7DyCKx8Mm6yAQAgJrIBACAmsgEAILY5Ho/P91dLH4MZ7faHaZqsvG5WHoGVR2DlEVh5BLv9wU02AADERDYAAMQ2x+Nx6TMAAMCquMkGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCA2Haapuf7q6WPwYx2+8Nk5bWz8gisPAIrj8DKI9jtD26yAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgtl36ALzL6+PDj5+vn14WPAnzsfIIrDwCK4/AyiP45MpusgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACC2XfoAvMv108vSR2B2Vh6BlUdg5RFYeQSfXNlNNgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMQ2x+Nx6TMAAMCquMkGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgNh2mqbNZvPLP/vy91+nPQyz2O0Pk5XXzsojsPIIrDwCK49gtz9sp9POeXd78+bvfP32/QQnOY3Xx4c3f+f66eUEJ7HyfKz8O1aeg5XnY+XfsfIcrDyfM1nZcxEAAIiJbAAAiIlsAACIbRf87J9f/7zntdCl+/n1z3teC62DlUdg5RFYeQRWHoGVT8ZNNgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADEtgt+9t3tzYKffnqvjw9LH2EBVh6BlUdg5RFYeQRWPhk32QAAEBPZAAAQE9kAABDbHI/H5/urpY/BjHb7wzRNVl43K4/AyiOw8gisPILd/uAmGwAAYiIbAABim+PxuPQZAABgVdxkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQExkAwBATGQDAEBMZAMAQGw7TdPz/dXSx2BGu/1hsvLaWXkEVh6BlUdg5RHs9gc32QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAENsufQDe5fXx4cfP108vC56E+Vh5BFYegZVHYOURfHJlN9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQ2y59AN7l+ull6SMwOyuPwMojsPIIrDyCT67sJhsAAGIiGwAAYp6LwLjubm9+/Pz12/cFTwIAK+MmG7gwu/1htz8sfQoA+BORDVwSeQ3ARRDZAAAQ8yYbYM28vAdYhJts4PI8318tfQR6XtsDa+ImG7gk8nqt5DWwMm6yAQAg5iYbAC6bl/dwhtxkA3AuPAdaJa/tGZObbACWJ6/XSl4zLDfZAAAQc5MNAHDuvLy/OG6yAYB5eQ60Sl7b/5mbbABgLvJ6reT1m9xkAwBATGQDAEBMZAMA8BGeA/2BN9kwLv8+HYCPkddvcpMNAAAxkQ0AADGRDQAAMW+yAdbMy3uARbjJBgCAmMgGAICYyAYAgJg32QBw2by8hzPkJhsAAGIiGwAAYiIbAABi3mQDAJw7L+8vjptsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiG2Ox+PSZwAAgFVxkw0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAse00TZvN5pd/9uXvv057GGax2x8mK6+dlUdg5RFYeQRWHsFuf9hOp53z7vbmzd/5+u37CU5yGq+PD2/+zvXTywlOYuX5WPl3rDwHK8/Hyr9j5TlYeT5nsrLnIgAAEBPZAAAQE9kAABDbLvjZP7/+ec9roUv38+uf97wWWgcrj8DKI7DyCKw8AiufjJtsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIhtF/zsu9ubBT/99F4fH5Y+wgKsPAIrj8DKI7DyCKx8Mm6yAQAgJrIBACAmsgEAILY5Ho/P91dLH4MZ7faHaZqsvG5WHoGVR2DlEVh5BLv9wU02AADERDYAAMQ2x+Nx6TMAAMCquMkGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCA2Haapuf7q6WPwYx2+8Nk5bWz8gisPAIrj8DKI9jtD26yAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgtl36ALzL6+PDj5+vn14WPAnzsfIIrDwCK4/AyiP45MpusgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACC2XfoAvMv108vSR2B2Vh6BlUdg5RFYeQSfXNlNNgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMS2H/jf3N3e/Pj567fv3WEAAGANLvgme7c/7PaHpU8BAAD/6VIjW14DAHC2LjWyAQDgbH3kTTYj8PIeAODDLvsm+/n+aukj0PPaHgC4dJd6ky2v10peAwArcNk32QAAcIYu9SYb+Dwv7wFgJm6yOUeeA62S1/YAjMNNNudFXq+VvAZgKG6yAQAg5iYbYM28vAdYhJts4HQ8B1olr+0B/pubbOAU5PVayWuAX3KTDQAAMTfZAHDZvLyHM+QmG4DP8hxolby2h8/4yE22vyUD8C/yeq3kNXySm2wAAIh5kw0AcO68vL84brIBgF/zHGiVvLY/DTfZ/Jq/JQOMTF6vlbw+GTfZAAAQE9kAABAT2QAAY/Ec6AS8yYZxeXkPMBp5fTJusgEAICayAQAgJrIBACDmTTbAmnl5D7AIN9kAABAT2QAAEBPZAAAQ8yYbAC6bl/dwhtxkAwBATGQDAEBMZAMAQMybbACAc+fl/cVxkw0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAACxzfF4XPoMAACwKm6yAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACC2naZps9n88s++/P3XaQ/DLHb7w2TltbPyCKw8AiuPwMoj2O0P2+m0c97d3rz5O1+/fT/BSU7j9fHhzd+5fno5wUmsPB8r/46V52Dl+Vj5d6w8ByvP50xW9lwEAABiIhsAAGIiGwAAYtsFP/vn1z/veS106X5+/fOe10LrYOURWHkEVh6BlUdg5ZNxkw0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAse2Cn313e7Pgp5/e6+PD0kdYgJVHYOURWHkEVh6BlU/GTTYAAMRENgAAxEQ2AADENsfj8fn+auljMKPd/jBNk5XXzcojsPIIrDwCK49gtz+4yQYAgJjIBgCA2OZ4PC59BgAAWBU32QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABDbTtP0fH+19DGY0W5/mKy8dlYegZVHYOURWHkEu/3BTTYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMS2Sx+Ad3l9fPjx8/XTy4InYT5WHoGVR2DlEVh5BJ9c2U02AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxLZLH4B3uX56WfoIzM7KI7DyCKw8AiuP4JMru8kGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCA2PYD/5u725sfP3/99r07DAAArMECN9m7/WG3P5z+cwEA4DROHdnyGgCA1fMmGwAAYh95k80IvLwHAPiwZW6yn++vFvlcZuW1PQDAv5z6Jlter5W8BgD4wZtsAACIeZMN4/LyHgBm4iabkudAq+S1PQD8U26yacjrtZLXAPABbrIBACDmJhtgzby8h3Xw3/LFcZMNvM1zoFXy2h4umv+Ez5ybbOBP5PVa+f9mgFm5yQYAuDD+nnz+3GQDwGXzWhfOkJtsgHF5DrRKnuqOw3/C5+wjN9n+lgxw6fx/81rJazgTnosAAFwYf08+f56LAABATGQDwNq45oTFeS7Cr3l5D3CJ5DWcCTfZAAAQE9kAABAT2QAAEPMmG8bl5T0AzMRNNgAAxEQ2AADERDYAAMS8yQZYMy/vYR38t3xx3GQDAEBMZAMAQExkAwBAzJtsALhsXuvCGXKTDQAAMZENAAAxkQ0AADGRDQAAsc3xeFz6DAAAsCpusgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgJrIBACAmsgEAICayAQAgtp2mabPZ/PLPvvz912kPwyx2+8Nk5bWz8gisPAIrj8DKI9jtD9vptHPe3d68+Ttfv30/wUlO4/Xx4c3fuX56OcFJrDwfK/+Oledg5flY+XesPAcrz+dMVvZcBAAAYiIbAABiIhsAAGLbBT/759c/73ktdOl+fv3zntdC62DlEVh5BFYegZVHYOWTcZMNAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AALHtgp99d3uz4Kef3uvjw9JHWICVR2DlEVh5BFYegZVPxk02AADERDYAAMRENgAAxDbH4/H5/mrpYzCj3f4wTZOV183KI7DyCKw8AiuPYLc/uMkGAICYyAYAgNjmeDwufQYAAFgVN9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQE9kAABAT2QAAEBPZAAAQ207T9Hx/tfQxmNFuf5isvHZWHoGVR2DlEVh5BLv9wU02AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADEtksfgHd5fXz48fP108uCJ2E+Vh6BlUdg5RFYeQSfXNlNNgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMRENgAAxEQ2AADERDYAAMS2Sx+Ad7l+eln6CMzOyiOw8gisPAIrj+CTK7vJBgCAmMgGAICYyAYAgJjIBgCAmMgGAICYyAYAgJjIBgCAmMgGABjXbn/Y7Q9Ln2KFRDYAAMRENgDAoNxhz0dkAwBATGQDAAzt+f5q6SOs0HbpAwAA8Ia725sfP3/99n3Bk/BOIhsAYFDusOfjuQgAAMRENgDwf3xrMiRENgAAxEQ2APBv7rChIrIBACAmsgGA/8c3TsDn+Qo/ALhsvkEZzpDIBgD+zR02VDwXAQCAmMgG4E98azLAB4hsAACIiWwAfssdNsDH+IePAGvmeycAFuEmG4A3+MYJgH9KZAMAQMxzEQB+yx02wMeIbACAc+ffVFwcz0WAf8a3JgPAm0Q2AADERDbwD7jDBoD38CYbxuUblAFgJm6ygX/MN04AwJ+JbAAAiHkuAvwD7rAB4D1ENgBcNv+mAs6Q55EQplcAAAFxSURBVCJ8lm9NBgD4D26y+TXfOwEA8GFusvkUd9gAAP9NZAMAQExkE/CNEwAAP/MmG2DN/JsKgEWIbD7FHTYAwH/zXAQAAGJ9ZPvWZAAABveR5yK+QRkAAP4gvsl2hw0AAN5kAwBAbJbI9o0TAACMzFf4wbj8mwoAmEkc2e6wAQDAm2wAAIiJbAAAiIlsAACIiWwAAIj5dhF+zfdOAAB8mJtsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIfeR7sn2DMgAA/IGbbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIiWwAAIiJbAAAiIlsAACIbY7H49JnAACAVXGTDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAMZENAAAxkQ0AADGRDQAAsf8FhEByBIOznGUAAAAASUVORK5CYII=",
      "text/plain": [
       "Console does not support images"
      ]
     },
     "metadata": {
      "image/png": {
       "height": 1272,
       "width": 972
      }
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(game_actions)\n",
    "screens = {}\n",
    "for i=1,36 do\n",
    "    screen, reward, terminal = game_env:step(0)\n",
    "    table.insert(screens, screen[1]:clone())\n",
    "end\n",
    "itorch.image(screens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dqn.NeuralQLearner\n",
       "{\n",
       "  lr_endt : 1000000\n",
       "  replay_memory : 1000000\n",
       "  bufferSize : 512\n",
       "  hist_len : 4\n",
       "  lr_start : 0.00025\n",
       "  histType : linear\n",
       "  bestq : 0\n",
       "  histSpacing : 1\n",
       "  deltas : CudaTensor - size: 1685667\n",
       "  input_dims : \n",
       "    {\n",
       "      1 : 4\n",
       "      2 : 84\n",
       "      3 : 84\n",
       "    }\n",
       "  n_hid : \n",
       "    {\n",
       "      1 : 512\n",
       "    }\n",
       "  ep_start : 1\n",
       "  q_max : 1\n",
       "  g : CudaTensor - size: 1685667\n",
       "  target_network : \n",
       "    nn.Sequential {\n",
       "      [input -> (1) -> (2) -> (3) -> (4) -> (5) -> (6) -> (7) -> (8) -> (9) -> (10) -> (11) -> output]\n",
       "      (1): nn.Reshape(4x84x84)\n",
       "      (2): nn.SpatialConvolution(in: 4, out: 32, kW: 8, kH: 8, dW: 4, dH: 4, padding: 1)\n",
       "      (3): nn.Rectifier\n",
       "      (4): nn.SpatialConvolution(in: 32, out: 64, kW: 4, kH: 4, dW: 2, dH: 2)\n",
       "      (5): nn.Rectifier\n",
       "      (6): nn.SpatialConvolution(in: 64, out: 64, kW: 3, kH: 3)\n",
       "      (7): nn.Rectifier\n",
       "      (8): nn.Reshape(3136)\n",
       "      (9): nn.Linear(3136 -> 512)\n",
       "      (10): nn.Rectifier\n",
       "      (11): nn.Linear(512 -> 3)\n",
       "    }\n",
       "    {\n",
       "      gradInput : CudaTensor - empty\n",
       "      modules : \n",
       "        {\n",
       "          1 : \n",
       "            nn.Reshape(4x84x84)\n",
       "            {\n",
       "              batchsize : LongStorage - size: 4\n",
       "              size : LongStorage - size: 3\n",
       "              output : CudaTensor - size: 1x4x84x84\n",
       "              gradInput : CudaTensor - empty\n",
       "              nelement : 28224\n",
       "              _gradOutput : CudaTensor - empty\n",
       "              _input : CudaTensor - empty\n",
       "            }\n",
       "          2 : \n",
       "            nn.SpatialConvolution(in: 4, out: 32, kW: 8, kH: 8, dW: 4, dH: 4, padding: 1)\n",
       "            {\n",
       "              dH : 4\n",
       "              dW : 4\n",
       "         "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "     nOutputPlane : 32\n",
       "              output : CudaTensor - size: 1x32x20x20\n",
       "              gradInput : CudaTensor - empty\n",
       "              finput : CudaTensor - empty\n",
       "              kW : 8\n",
       "              kH : 8\n",
       "              weight : CudaTensor - size: 32x4x8x8\n",
       "              nInputPlane : 4\n",
       "              gradWeight : CudaTensor - size: 32x4x8x8\n",
       "              padding : 1\n",
       "              bias : CudaTensor - size: 32\n",
       "              gradBias : CudaTensor - size: 32\n",
       "              fgradInput : CudaTensor - empty\n",
       "            }\n",
       "          3 : \n",
       "            nn.Rectifier\n",
       "            {\n",
       "              gradInput : CudaTensor - empty\n",
       "              output : CudaTensor - size: 1x32x20x20\n",
       "            }\n",
       "          4 : \n",
       "            nn.SpatialConvolution(in: 32, out: 64, kW: 4, kH: 4, dW: 2, dH: 2)\n",
       "            {\n",
       "              dH : 2\n",
       "              dW : 2\n",
       "              nOutputPlane : 64\n",
       "              output : CudaTensor - size: 1x64x9x9\n",
       "              gradInput : CudaTensor - empty\n",
       "              finput : CudaTensor - empty\n",
       "              kW : 4\n",
       "              kH : 4\n",
       "              weight : CudaTensor - size: 64x32x4x4\n",
       "              nInputPlane : 32\n",
       "              gradWeight : CudaTensor - size: 64x32x4x4\n",
       "              padding : 0\n",
       "              bias : CudaTensor - size: 64\n",
       "              gradBias : CudaTensor - size: 64\n",
       "              fgradInput : CudaTensor - empty\n",
       "       "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "     }\n",
       "          5 : \n",
       "            nn.Rectifier\n",
       "            {\n",
       "              gradInput : CudaTensor - empty\n",
       "              output : CudaTensor - size: 1x64x9x9\n",
       "            }\n",
       "          6 : \n",
       "            nn.SpatialConvolution(in: 64, out: 64, kW: 3, kH: 3)\n",
       "            {\n",
       "              dH : 1\n",
       "              dW : 1\n",
       "              nOutputPlane : 64\n",
       "              output : CudaTensor - size: 1x64x7x7\n",
       "              gradInput : CudaTensor - empty\n",
       "              finput : CudaTensor - empty\n",
       "              kW : 3\n",
       "              kH : 3\n",
       "              weight : CudaTensor - size: 64x64x3x3\n",
       "              nInputPlane : 64\n",
       "              gradWeight : CudaTensor - size: 64x64x3x3\n",
       "              padding : 0\n",
       "              bias : CudaTensor - size: 64\n",
       "              gradBias : CudaTensor - size: 64\n",
       "              fgradInput : CudaTensor - empty\n",
       "            }\n",
       "          7 : \n",
       "            nn.Rectifier\n",
       "            {\n",
       "              gradInput : CudaTensor - empty\n",
       "              output : CudaTensor - size: 1x64x7x7\n",
       "            }\n",
       "          8 : \n",
       "            nn.Reshape(3136)\n",
       "            {\n",
       "              batchsize : LongStorage - size: 2\n",
       "              size : LongStorage - size: 1\n",
       "              output : CudaTensor - empty\n",
       "              gradInput : CudaTensor - empty\n",
       "              nelement : 3136\n",
       "              _gradOutput : CudaTensor - empty\n",
       "              _input : CudaTensor - empty\n",
       "            }\n",
       "          9 : \n",
       "            nn.Linear(3136 -> 512)\n",
       "            {\n",
       "              gradBias : CudaTensor - size: 512\n",
       "              weight : CudaTensor - size: 512x3136\n",
       "              bias : CudaTensor - size: 512\n",
       "              gradInput : CudaTensor - empty\n",
       "              output : CudaTensor - empty\n",
       "              gradWeight : CudaTensor - size: 512x3136\n",
       "            }\n",
       "        "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "  10 : \n",
       "            nn.Rectifier\n",
       "            {\n",
       "              gradInput : CudaTensor - empty\n",
       "              output : CudaTensor - empty\n",
       "            }\n",
       "          11 : \n",
       "            nn.Linear(512 -> 3)\n",
       "            {\n",
       "              gradBias : CudaTensor - size: 3\n",
       "              weight : CudaTensor - size: 3x512\n",
       "              bias : CudaTensor - size: 3\n",
       "              gradInput : CudaTensor - empty\n",
       "              output : CudaTensor - empty\n",
       "              gradWeight : CudaTensor - size: 3x512\n",
       "            }\n",
       "        }\n",
       "      output : CudaTensor - empty\n",
       "    }\n",
       "  ep_end : 0.1\n",
       "  g2 : CudaTensor - size: 1685667\n",
       "  w : CudaTensor - size: 1685667\n",
       "  update_freq : 4\n",
       "  minibatch_size : 32\n",
       "  ncols : 1\n",
       "  r_max : 1\n",
       "  transition_params : table: 0x41621680\n",
       "  lr_end : 0.00025\n",
       "  target_q : 10000\n",
       "  numSteps : 0\n",
       "  v_avg : 0\n",
       "  n_units : \n",
       "    {\n",
       "      1 : 32\n",
       "      2 : 64\n",
       "      3 : 64\n",
       "    }\n",
       "  ep_endt : 1000000\n",
       "  min_reward : -1\n",
       "  discount : 0.99\n",
       "  learn_start : 50000\n",
       "  filter_stride : \n",
       "    {\n",
       "      1 : 4\n",
       "      2 : 2\n",
       "      3 : 1\n",
       "    }\n",
       "  ep : 1\n",
       "  tderr_avg : 0\n",
       "  transitions : \n",
       "    dqn.TransitionTable\n",
       "    {\n",
       "      stateDim : 7056\n",
       "      histLen : 4\n",
       "      recentMemSize : 4\n",
       "      nonEventProb : 1\n",
       "      maxSize : 1000000\n",
       "      bufferSize : 512\n",
       "      a : LongTensor - size: 1000000\n",
       "      gpu_s2 : CudaTensor - size: 512x28224\n",
       "      histType : linear\n",
       "      gpu_s : CudaTensor - size: 512x28224\n",
       "      insertIndex : 0\n",
       "      histSpacing : 1\n",
       "      buf_s : ByteTensor - size: 512x28224\n",
       "      buf_s2 : ByteTensor - size: 512x28224\n",
       "      gpu : 0\n",
       "      numActions : 3\n",
       "      nonTermProb : 1\n",
       "      s : ByteTensor - size: 1000000x7056\n",
       "      action_encodings : FloatTensor - size: 3x3\n",
       "      buf_term : ByteTensor - size: 512\n",
       "      buf_r : FloatTensor - size: 512\n",
       "      t : ByteTensor - size: 1000000\n",
       "      buf_a : LongTensor - size: 512\n",
       "      recent_t : table: 0x41d7e5b8\n",
       "      recent_a : table: 0x41d7e590\n",
       "      recent_s : table: 0x41d7e568\n",
       "      r : FloatTensor - size: 1000000\n",
       "      zeroFrames : 1\n",
       "      numEntries : 0\n"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "      histIndices : \n",
       "        {\n",
       "          1 : 1\n",
       "          2 : 2\n",
       "          3 : 3\n",
       "          4 : 4\n",
       "        }\n",
       "    }\n",
       "  max_reward : 1\n",
       "  wc : 0\n",
       "  tensor_type : table: 0x40d6f718\n",
       "  nl : table: 0x41d65620\n",
       "  actions : \n",
       "    {\n",
       "      1 : 0\n",
       "      2 : 3\n",
       "      3 : 4\n",
       "    }\n",
       "  n_replay : 1\n",
       "  nonTermProb : 1\n",
       "  tmp : CudaTensor - size: 1685667\n",
       "  network : \n",
       "    nn.Sequential {\n",
       "      [input -> (1) -> (2) -> (3) -> (4) -> (5) -> (6) -> (7) -> (8) -> (9) -> (10) -> (11) -> output]\n",
       "      (1): nn.Reshape(4x84x84)\n",
       "      (2): nn.SpatialConvolution(in: 4, out: 32, kW: 8, kH: 8, dW: 4, dH: 4, padding: 1)\n",
       "      (3): nn.Rectifier\n",
       "      (4): nn.SpatialConvolution(in: 32, out: 64, kW: 4, kH: 4, dW: 2, dH: 2)\n",
       "      (5): nn.Rectifier\n",
       "      (6): nn.SpatialConvolution(in: 64, out: 64, kW: 3, kH: 3)\n",
       "      (7): nn.Rectifier\n",
       "      (8): nn.Reshape(3136)\n",
       "      (9): nn.Linear(3136 -> 512)\n",
       "      (10): nn.Rectifier\n",
       "      (11): nn.Linear(512 -> 3)\n",
       "    }\n",
       "    {\n",
       "      gradInput : CudaTensor - empty\n",
       "      modules : \n",
       "        {\n",
       "          1 : \n",
       "            nn.Reshape(4x84x84)\n",
       "            {\n",
       "              nelement : 28224\n",
       "              _input : CudaTensor - empty\n",
       "              output : CudaTensor - size: 1x4x84x84\n",
       "              gradInput : CudaTensor - empty\n",
       "              size : LongStorage - size: 3\n",
       "              _gradOutput : CudaTensor - empty\n",
       "              batchsize : LongStorage - size: 4\n",
       "            }\n",
       "          2 : \n",
       "            nn.SpatialConvolution(in: 4, out: 32, kW: 8, kH: 8, dW: 4, dH: 4, padding: 1)\n",
       "            {\n",
       "              dH : 4\n",
       "              dW : 4\n",
       "              nOutputPlane : 32\n",
       "              output : CudaTensor - size: 1x32x20x20\n",
       "              gradInput : CudaTensor - empty\n",
       "              finput : CudaTensor - empty\n",
       "              fgradInput : CudaTensor - empty\n",
       "              gradBias : CudaTensor - size: 32\n",
       "              weight : CudaTensor - size: 32x4x8x8\n",
       "              bias : CudaTensor - size: 32\n",
       "              gradWeight : CudaTensor - size: 32x4x8x8\n",
       "              padding : 1\n",
       "              nInputPlane : 4\n",
       "              kW : 8\n",
       "              kH : 8\n",
       "            }\n",
       " "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "         3 : \n",
       "            nn.Rectifier\n",
       "            {\n",
       "              gradInput : CudaTensor - empty\n",
       "              output : CudaTensor - size: 1x32x20x20\n",
       "            }\n",
       "          4 : \n",
       "            nn.SpatialConvolution(in: 32, out: 64, kW: 4, kH: 4, dW: 2, dH: 2)\n",
       "            {\n",
       "              dH : 2\n",
       "              dW : 2\n",
       "              nOutputPlane : 64\n",
       "              output : CudaTensor - size: 1x64x9x9\n",
       "              gradInput : CudaTensor - empty\n",
       "              finput : CudaTensor - empty\n",
       "              fgradInput : CudaTensor - empty\n",
       "              gradBias : CudaTensor - size: 64\n",
       "              weight : CudaTensor - size: 64x32x4x4\n",
       "              bias : CudaTensor - size: 64\n",
       "              gradWeight : CudaTensor - size: 64x32x4x4\n",
       "              padding : 0\n",
       "              nInputPlane : 32\n",
       "              kW : 4\n",
       "              kH : 4\n",
       "            }\n",
       "          5 : \n",
       "            nn.Rectifier\n",
       "            {\n",
       "              gradInput : CudaTensor - empty\n",
       "              output : CudaTensor - size: 1x64x9x9\n",
       "            }\n",
       "          6 : \n",
       "            nn.SpatialConvolution(in: 64, out: 64, kW: 3, kH: 3)\n",
       "            {\n",
       "              dH : 1\n",
       "              dW : 1\n",
       "              nOutputPlane : 64\n",
       "              output : CudaTensor - size: 1x64x7x7\n",
       "              gradInput : CudaTensor - empty\n",
       "              finput : CudaTensor - empty\n",
       "              fgradInput : CudaTensor - empty\n",
       "              gradBias : CudaTensor - size: 64\n",
       "              weight : CudaTensor - size: 64x64x3x3\n",
       "              bias : CudaTensor - size: 64\n"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "              gradWeight : CudaTensor - size: 64x64x3x3\n",
       "              padding : 0\n",
       "              nInputPlane : 64\n",
       "              kW : 3\n",
       "              kH : 3\n",
       "            }\n",
       "          7 : \n",
       "            nn.Rectifier\n",
       "            {\n",
       "              gradInput : CudaTensor - empty\n",
       "              output : CudaTensor - size: 1x64x7x7\n",
       "            }\n",
       "          8 : \n",
       "            nn.Reshape(3136)\n",
       "            {\n",
       "              nelement : 3136\n",
       "              _input : CudaTensor - empty\n",
       "              output : CudaTensor - empty\n",
       "              gradInput : CudaTensor - empty\n",
       "              size : LongStorage - size: 1\n",
       "              _gradOutput : CudaTensor - empty\n",
       "              batchsize : LongStorage - size: 2\n",
       "            }\n",
       "          9 : \n",
       "            nn.Linear(3136 -> 512)\n",
       "            {\n",
       "              gradBias : CudaTensor - size: 512\n",
       "              weight : CudaTensor - size: 512x3136\n",
       "              bias : CudaTensor - size: 512\n",
       "              gradInput : CudaTensor - empty\n",
       "              gradWeight : CudaTensor - size: 512x3136\n",
       "              output : CudaTensor - empty\n",
       "            }\n",
       "          10 : \n",
       "            nn.Rectifier\n",
       "            {\n",
       "              gradInput : CudaTensor - empty\n",
       "              output : CudaTensor - empty\n",
       "            }\n",
       "          11 : \n",
       "            nn.Linear(512 -> 3)\n",
       "            {\n",
       "              gradBias : CudaTensor - size: 3\n",
       "              weight : CudaTensor - size: 3x512\n",
       "              bias : CudaTensor - size: 3\n",
       "              gradInput : CudaTensor - empty\n",
       "              gradWeight : CudaTensor - size: 3x512\n",
       "              output : CudaTensor - empty\n",
       "            }\n",
       "        }\n",
       "      "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "output : CudaTensor - empty\n",
       "    }\n",
       "  gpu : 0\n",
       "  filter_size : \n",
       "    {\n",
       "      1 : 8\n",
       "      2 : 4\n",
       "      3 : 3\n",
       "    }\n",
       "  lr : 0.00025\n",
       "  preproc : \n",
       "    nn.Scale\n",
       "    {\n",
       "      height : 84\n",
       "      width : 84\n",
       "    }\n",
       "  valid_size : 500\n",
       "  clip_delta : 1\n",
       "  verbose : 2\n",
       "  rescale_r : 1\n",
       "  n_actions : 3\n",
       "  state_dim : 7056\n",
       "  dw : CudaTensor - size: 1685667\n",
       "}\n"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "agent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Iteration ..\t0\t\n"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "Training Steps: \t10000\t\n"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "local learn_start = agent.learn_start\n",
    "local start_time = sys.clock()\n",
    "local reward_counts = {}\n",
    "local episode_counts = {}\n",
    "local time_history = {}\n",
    "local v_history = {}\n",
    "local qmax_history = {}\n",
    "local td_history = {}\n",
    "local reward_history = {}\n",
    "local step = 0\n",
    "time_history[1] = 0\n",
    "\n",
    "local total_reward\n",
    "local nrewards\n",
    "local nepisodes\n",
    "local episode_reward\n",
    "\n",
    "\n",
    "-- When we save agent, there are many temporary states in the agent. The proper way would be to implement \n",
    "-- agent:read() and agent:write() functions as in TransitionTable.lua \n",
    "-- But using Lua closures, we can also have a nice interface like follows.\n",
    "-- agent is the agent to make slim so that we do not dump GBs of data\n",
    "-- returns a deslim function that does not take any arguments and reinstates the parameters in the agent.\n",
    "local function slim_agent(agent)\n",
    "    local s, a, r, s2, term = agent.valid_s, agent.valid_a, agent.valid_r,agent.valid_s2, agent.valid_term\n",
    "    agent.valid_s, agent.valid_a, agent.valid_r, agent.valid_s2, agent.valid_term = nil, nil, nil, nil, nil, nil, nil\n",
    "    local w, dw, g, g2, delta, delta2, deltas, tmp = agent.w, agent.dw, agent.g, agent.g2, agent.delta, agent.delta2, agent.deltas, agent.tmp\n",
    "    agent.w, agent.dw, agent.g, agent.g2, agent.delta, agent.delta2, agent.deltas, agent.tmp = nil, nil, nil, nil, nil, nil, nil, nil\n",
    "    \n",
    "    -- this function now has pointers to all the local variables here when it was called.\n",
    "    -- so it can remodify the agent with the correct values. One could call slim_agent with many different\n",
    "    -- agent instances, and every deslim function would point to the correct context.\n",
    "    local function deslim()\n",
    "        agent.valid_s, agent.valid_a, agent.valid_r, agent.valid_s2, agent.valid_term = s, a, r, s2, term\n",
    "        agent.w, agent.dw, agent.g, agent.g2, agent.delta, agent.delta2, agent.deltas, agent.tmp = w, dw, g, g2, delta, delta2, deltas, tmp\n",
    "    end\n",
    "    return deslim\n",
    "end\n",
    "\n",
    "local screen, reward, terminal = game_env:getState()\n",
    "\n",
    "print(\"Iteration ..\", step)\n",
    "while step < opt.steps do\n",
    "    step = step + 1\n",
    "    local action_index = agent:perceive(reward, screen, terminal)\n",
    "\n",
    "    -- game over? get next game!                                                                                                                      \n",
    "    if not terminal then\n",
    "        screen, reward, terminal = game_env:step(game_actions[action_index], true)\n",
    "    else\n",
    "        if opt.random_starts > 0 then\n",
    "            screen, reward, terminal = game_env:nextRandomGame()\n",
    "        else\n",
    "            screen, reward, terminal = game_env:newGame()\n",
    "        end\n",
    "    end\n",
    "\n",
    "    if step % opt.prog_freq == 0 then\n",
    "        assert(step==agent.numSteps, 'trainer step: ' .. step ..\n",
    "                ' & agent.numSteps: ' .. agent.numSteps)\n",
    "        print(\"Training Steps: \", step)\n",
    "        --agent:report()\n",
    "        collectgarbage()\n",
    "    end\n",
    "\n",
    "    if step%1000 == 0 then collectgarbage() end\n",
    "\n",
    "    if step % opt.eval_freq == 0 and step > learn_start then\n",
    "\n",
    "        print('Evaluating')\n",
    "        screen, reward, terminal = game_env:newGame()\n",
    "\n",
    "        total_reward = 0\n",
    "        nrewards = 0\n",
    "        nepisodes = 0\n",
    "        episode_reward = 0\n",
    "\n",
    "        local eval_time = sys.clock()\n",
    "        for estep=1,opt.eval_steps do\n",
    "            local action_index = agent:perceive(reward, screen, terminal, true, 0.05)\n",
    "            -- Play game in test mode (episodes don't end when losing a life)                                                                         \n",
    "            screen, reward, terminal = game_env:step(game_actions[action_index])\n",
    "\n",
    "            if estep%1000 == 0 then collectgarbage() end\n",
    "\n",
    "            -- record every reward                                                                                                                    \n",
    "            episode_reward = episode_reward + reward\n",
    "            if reward ~= 0 then\n",
    "               nrewards = nrewards + 1\n",
    "            end\n",
    "\n",
    "            if terminal then\n",
    "                total_reward = total_reward + episode_reward\n",
    "                episode_reward = 0\n",
    "                nepisodes = nepisodes + 1\n",
    "                screen, reward, terminal = game_env:nextRandomGame()\n",
    "            end\n",
    "        end\n",
    "\n",
    "        eval_time = sys.clock() - eval_time\n",
    "        start_time = start_time + eval_time\n",
    "        agent:compute_validation_statistics()\n",
    "        local ind = #reward_history+1\n",
    "        total_reward = total_reward/math.max(1, nepisodes)\n",
    "\n",
    "        if #reward_history == 0 or total_reward > torch.Tensor(reward_history):max() then\n",
    "            agent.best_network = agent.network:clone()\n",
    "        end\n",
    "\n",
    "        if agent.v_avg then\n",
    "            v_history[ind] = agent.v_avg\n",
    "            td_history[ind] = agent.tderr_avg\n",
    "            qmax_history[ind] = agent.q_max\n",
    "        end\n",
    "        print(\"V\", v_history[ind], \"TD error\", td_history[ind], \"Qmax\", qmax_history[ind])\n",
    "\n",
    "        reward_history[ind] = total_reward\n",
    "        reward_counts[ind] = nrewards\n",
    "        episode_counts[ind] = nepisodes\n",
    "        time_history[ind+1] = sys.clock() - start_time\n",
    "\n",
    "        local time_dif = time_history[ind+1] - time_history[ind]\n",
    "\n",
    "        local training_rate = opt.actrep*opt.eval_freq/time_dif\n",
    "\n",
    "        print(string.format(\n",
    "            '\\nSteps: %d (frames: %d), reward: %.2f, epsilon: %.2f, lr: %G, ' ..\n",
    "            'training time: %ds, training rate: %dfps, testing time: %ds, ' ..\n",
    "            'testing rate: %dfps,  num. ep.: %d,  num. rewards: %d',\n",
    "            step, step*opt.actrep, total_reward, agent.ep, agent.lr, time_dif,\n",
    "            training_rate, eval_time, opt.actrep*opt.eval_steps/eval_time,\n",
    "            nepisodes, nrewards))\n",
    "    end\n",
    "\n",
    "    if step % opt.save_freq == 0 or step == opt.steps then\n",
    "        \n",
    "        local deslim_agent = slim_agent(agent)\n",
    "\n",
    "        local filename = opt.name\n",
    "        if opt.save_versions > 0 then\n",
    "            filename = filename .. \"_\" .. math.floor(step / opt.save_versions)\n",
    "        end\n",
    "        filename = filename\n",
    "        torch.save(filename .. \".t7\", {agent = agent,\n",
    "                                model = agent.network,\n",
    "                                best_model = agent.best_network,\n",
    "                                reward_history = reward_history,\n",
    "                                reward_counts = reward_counts,\n",
    "                                episode_counts = episode_counts,\n",
    "                                time_history = time_history,\n",
    "                                v_history = v_history,\n",
    "                                td_history = td_history,\n",
    "                                qmax_history = qmax_history,\n",
    "                                arguments=opt})\n",
    "        if opt.saveNetworkParams then\n",
    "            local nets = {network=w:clone():float()}\n",
    "            torch.save(filename..'.params.t7', nets, 'ascii')\n",
    "        end\n",
    "        deslim_agent()\n",
    "        print('Saved:', filename .. '.t7')\n",
    "        io.flush()\n",
    "        collectgarbage()\n",
    "    end\n",
    "end\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "iTorch",
   "language": "lua",
   "name": "itorch"
  },
  "language_info": {
   "name": "lua",
   "version": "5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}