Maze2D_Getting_the_transition_model.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/yafo1948/Reinforcement_Learning_RUNI.TASHPAG/blob/main/Maze2D_Getting_the_transition_model.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Installs"
      ],
      "metadata": {
        "id": "VH9ZX_RT8eFo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%%capture\n",
        "!sudo apt-get update\n",
        "!sudo apt-get install -y xvfb ffmpeg freeglut3-dev\n",
        "!pip install 'imageio==2.4.0'\n",
        "!pip install gym\n",
        "!pip install pygame\n",
        "!apt-get install python-opengl -y\n",
        "!apt install xvfb -y\n",
        "!pip install pyvirtualdisplay\n",
        "!pip install piglet\n",
        "!pip install -U --no-cache-dir gdown --pre\n",
        "!gdown --id 1FeuIx5OVLmfCx0dxxwU-7Xn8gpPc-53D\n",
        "!unzip /content/maze_mid.zip"
      ],
      "metadata": {
        "id": "xuJdLI_YPAvD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Imports"
      ],
      "metadata": {
        "id": "18TKb2kW6MKc"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NbxNfCQq2-8H"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import gym\n",
        "from gym import logger as gymlogger\n",
        "from gym.utils import seeding\n",
        "from gym import error, spaces, utils\n",
        "gymlogger.set_level(40) # error only\n",
        "import glob\n",
        "import io\n",
        "import base64\n",
        "import os\n",
        "import random\n",
        "import matplotlib.pyplot as plt\n",
        "%matplotlib inline\n",
        "import math\n",
        "import glob\n",
        "from pyvirtualdisplay import Display\n",
        "from IPython.display import HTML\n",
        "from IPython import display as ipythondisplay\n",
        "import pygame\n",
        "from maze_mid import *\n",
        "from maze_mid.cust_maze import MazeEnvCast5x5, MazeEnvCast15x15, MazeEnvCast25x25\n",
        "import pyvirtualdisplay\n",
        "import imageio\n",
        "import IPython\n",
        "import time"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def embed_mp4(filename):\n",
        "  \"\"\"Embeds an mp4 file in the notebook.\"\"\"\n",
        "  video = open(filename,'rb').read()\n",
        "  b64 = base64.b64encode(video)\n",
        "  tag = '''\n",
        "  <video width=\"640\" height=\"480\" controls>\n",
        "    <source src=\"data:video/mp4;base64,{0}\" type=\"video/mp4\">\n",
        "  Your browser does not support the video tag.\n",
        "  </video>'''.format(b64.decode())\n",
        "\n",
        "  return IPython.display.HTML(tag)\n",
        "display = pyvirtualdisplay.Display(visible=0, size=(1400, 900)).start()"
      ],
      "metadata": {
        "id": "Vfoo4W9G4nMM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Helpers and utilities**\n",
        "The next cell holds a function that verifies which actions are available from your current state and a mapping from actions to logical names, directions and movements."
      ],
      "metadata": {
        "id": "rUvTW-_Z5M7x"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Numbers to action mapping.\n",
        "nTa = {\n",
        "       0:(\"UP\",\"N\", (0, -1)),\n",
        "       1:(\"DOWN\",\"S\", (0, 1)),\n",
        "       2:(\"RIGHT\",\"E\", (1, 0)),\n",
        "       3:(\"LEFT\",\"W\",(-1, 0))\n",
        "       };\n",
        "\n",
        "def get_available_actions(state):\n",
        "    available_actions = [];\n",
        "    for action in actions:\n",
        "        # Get the correct letter of the action.\n",
        "        action_letter = nTa[action][1];\n",
        "        \n",
        "        # Check if the move is legit.\n",
        "        legit = env.unwrapped.maze_view.maze.is_open(state,action_letter);\n",
        "        if(legit): available_actions.append(action);\n",
        "\n",
        "    return available_actions;      "
      ],
      "metadata": {
        "id": "aX3X4v0g5MWY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "This section demonstrates how to artificially generate the transition model\n",
        "for the Maze game which does not implicitly supply it.\n",
        "We also use our own custom logic to make the actions stochastic."
      ],
      "metadata": {
        "id": "kc4DLMxh4N9E"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Initialize the environment.\n",
        "env = MazeEnvCast5x5();\n",
        "\n",
        "# --------------------------------------------- #\n",
        "# Get the basic information of the environment.\n",
        "# --------------------------------------------- #\n",
        "\n",
        "# Number of states.\n",
        "number_of_states = np.prod((env.observation_space.high - env.observation_space.low) + 1 );\n",
        "\n",
        "# Define the terminal state.\n",
        "terminal_state = (env.observation_space.high[0],env.observation_space.high[1]);\n",
        "\n",
        "# Actions.\n",
        "actions = np.arange(env.action_space.n);\n",
        "\n",
        "# Maze rows and cols.\n",
        "rows = env.observation_space.high[0] + 1;\n",
        "cols = env.observation_space.high[1] + 1;"
      ],
      "metadata": {
        "id": "LC6mv2Fu4NiM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get the maze as a matrix.\n",
        "maze = env.unwrapped.maze_view.maze.maze_cells;\n",
        "\n",
        "print(\"The maze is:\\n\",maze);"
      ],
      "metadata": {
        "id": "LaJyIFHi48lq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Initialize R - The rewards.\n",
        "R = np.ones_like(maze,dtype = float) * -(0.1/number_of_states);\n",
        "R[terminal_state] = 1;\n",
        "\n",
        "print(R);"
      ],
      "metadata": {
        "id": "3-tnVPkP51nl"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Our custom transition probability.\n",
        "p = 0.9;\n",
        "\n",
        "# 2. Create a placeholder for the model P which mimics env.P.\n",
        "P = {};\n",
        "for i in range(rows):\n",
        "    for j in range(cols):\n",
        "            P.update({(i,j): {action : [] for action in actions}});\n",
        "\n",
        "# Create the model.\n",
        "for state in P.keys():\n",
        "    # Get the available actions.\n",
        "    a_actions = get_available_actions(state);\n",
        "    # Get the states.\n",
        "    next_states = [];\n",
        "    \n",
        "    # The action the agent chose.\n",
        "    for chosen_action in actions:\n",
        "        # The action the agent actually takes.\n",
        "        for actual_action in actions:\n",
        "            # Find out if youre staying in place or actually moving.\n",
        "            next_state = tuple(np.add(state,nTa[actual_action][2])) if (actual_action in a_actions) else state;\n",
        "            # Append the transition model.\n",
        "            if(actual_action == chosen_action):\n",
        "                P[state][chosen_action].append([p,next_state,R[next_state],0]);\n",
        "            else:\n",
        "                P[state][chosen_action].append([(1 - p) / 3,next_state,R[next_state],0]);"
      ],
      "metadata": {
        "id": "QdAVLPD55IF7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Treating P as env.P**\n",
        "\n",
        "We can now use P just like we would use env.P"
      ],
      "metadata": {
        "id": "Qp8nEuF16Fvo"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Create the uniform random policy.\n",
        "π = {state:np.ones_like(actions) / len(actions) for state in P.keys()};\n",
        "\n",
        "# Observe the initial position.\n",
        "state = (0,0);\n",
        "\n",
        "for action,action_probability in enumerate(π[state]):\n",
        "  print(action,action_probability)\n",
        "  for transition_probability, next_state, reward, done in P[state][action]:\n",
        "    print(transition_probability, next_state, reward, done);"
      ],
      "metadata": {
        "id": "aTEtXi4L6FPe"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}