diff --git a/README-v2.md b/README-v2.md index 5e55af6..2ad4a11 100644 --- a/README-v2.md +++ b/README-v2.md @@ -55,6 +55,9 @@ while True: rewards += reward if done or truncated: break + +# You can also get the map of the rooms +room_layout = env.unwrapped.return_room_layout(exclude_walls=True) ``` Take a look at [this repo](https://github.com/humemai/humemai) for an actual diff --git a/README.md b/README.md index 62c4d8e..8e8f46a 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,6 @@ repo](https://github.com/humemai/humemai). - ["A Machine With Human-Like Memory Systems"](https://arxiv.org/abs/2204.01611). - ["A Machine with Short-Term, Episodic, and Semantic Memory Systems"](https://doi.org/10.1609/aaai.v37i1.25075). -- ["Capturing Dynamic Knowledge Graphs with Human-like Memory Systems by Reinforcement - Learning"](<>). ## pdoc documentation diff --git a/miscellaneous/Capturing dynamic knowledge graphs with human-like memory systems by reinforcement learning.md b/miscellaneous/Capturing dynamic knowledge graphs with human-like memory systems by reinforcement learning.md deleted file mode 100644 index 3e7853f..0000000 --- a/miscellaneous/Capturing dynamic knowledge graphs with human-like memory systems by reinforcement learning.md +++ /dev/null @@ -1,37 +0,0 @@ -# [Capturing dynamic knowledge graphs with human-like memory systems by reinforcement learning](https://www.overleaf.com/9972424828hxktdqjxnmsw) - -## Now the environment has more than one room. - -- This adds one more action space. That is, the agent has to decide which room to go at - time $t$. In that room, the initial action space follows, where it has to decide in which - memory system it should store its observation. - -## Observations can also be extended. - -- Instead of simple triples, (one’s object, atlocation, furniture), it can include more things. -- This means that the questions will also become more complicated. Instead of simply asking the object location, it can also ask other things. - -## Delayed rewards - -- Currently, a question is asked every time the agent takes an action. This is not realistic. Questions can be asked at any time, leading to delayed rewards. -- Actually, this is already done. Delayed rewards work fine. - -## Other things to consider - -- LSTM might not be a suitable function apprximator any more here, but a more generic GNN might be a better option. -- DQN still might work well, but other RL algos might work better. - -## The three policies - -1. Question answering policy: $\\pi\_{qa}(a\_{qa}|M\_{long})$ -1. Memory management policy: $\\pi\_{memory}(a\_{memory} | M\_{short}, M\_{long})$ -1. Exploration policy: $\\pi\_{explore}(a\_{explore} | M\_{long})$ - -First, I'll have the agent learn one policy, while the other two are fixed. How can I do -this actually? Should this be one environment? - -## The knowledge graphs - -- Both the hidden states and the observed sub-graphs are knowledge graphs but they are - still quite too simple to be called a knowledge graph. They only have one relation. - How can I add more relations? diff --git a/miscellaneous/pomdp/Homework from Michael_221018_180939.jpg b/miscellaneous/pomdp/Homework from Michael_221018_180939.jpg deleted file mode 100644 index 26725ac..0000000 Binary files a/miscellaneous/pomdp/Homework from Michael_221018_180939.jpg and /dev/null differ diff --git a/miscellaneous/pomdp/pomdp.ipynb b/miscellaneous/pomdp/pomdp.ipynb deleted file mode 100644 index f7be4ba..0000000 --- a/miscellaneous/pomdp/pomdp.ipynb +++ /dev/null @@ -1,1856 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "class Foo:\n", - " def __init__(self, bar: str) -> None:\n", - " self.bar = bar\n", - "\n", - " def my_method(self, baz: int) -> int:\n", - " breakpoint()\n", - " print(baz)\n", - " return baz\n", - "\n", - "\n", - "foo = Foo(\"asdf\")" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "1.22" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "foo.my_method(1.22)" - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [], - "source": [ - "import random\n", - "\n", - "\n", - "class Object:\n", - " def __init__(\n", - " self,\n", - " name: str,\n", - " type: str,\n", - " ) -> None:\n", - " \"\"\"Entity, e.g., human, object, room.\n", - "\n", - " Args\n", - " ----\n", - " name: e.g., Tae, laptop, bed\n", - " type: e.g., static, independent, dependent\n", - " \"\"\"\n", - " self.name = name\n", - " self.type = type\n", - "\n", - " def __repr__(self) -> str:\n", - " return f\"Object({self.name}, {self.type})\"\n", - "\n", - " def move(self) -> None:\n", - " \"\"\"Move object to another room.\"\"\"\n", - " pass\n", - "\n", - " def step(self) -> None:\n", - " \"\"\"Move object to another room.\"\"\"\n", - " pass\n", - "\n", - "\n", - "class StaticObject(Object):\n", - " def __init__(self, name: str, location: str) -> None:\n", - " super().__init__(name, \"static\")\n", - " self.location = location\n", - "\n", - " def __repr__(self) -> str:\n", - " return f\"StaticObject({self.name}, {self.location})\"\n", - "\n", - " def move(self) -> None:\n", - " \"\"\"Static object cannot move.\"\"\"\n", - " raise ValueError(\"Static object cannot move.\")\n", - "\n", - " def step(self) -> None:\n", - " \"\"\"Static object cannot move.\"\"\"\n", - " pass\n", - "\n", - "\n", - "class IndepdentObject(Object):\n", - " def __init__(self, name: str, probs: dict) -> None:\n", - " super().__init__(name, \"independent\")\n", - " self.probs = probs\n", - " self.move()\n", - "\n", - " def __repr__(self) -> str:\n", - " return f\"IndependentObject({self.name}, {self.location})\"\n", - "\n", - " def move(self) -> None:\n", - " \"\"\"Indendent object objects to another room.\"\"\"\n", - " self.location = random.choices(\n", - " list(self.probs.keys()),\n", - " weights=list(self.probs.values()),\n", - " k=1,\n", - " )[0]\n", - "\n", - " def step(self) -> None:\n", - " \"\"\"Indendent object objects to another room.\"\"\"\n", - " self.location = random.choices(\n", - " list(self.probs.keys()),\n", - " weights=list(self.probs.values()),\n", - " k=1,\n", - " )[0]\n", - "\n", - "\n", - "class DependentObject(Object):\n", - " def __init__(\n", - " self, name: str, dependence: list[tuple[IndepdentObject, float]]\n", - " ) -> None:\n", - " super().__init__(name, \"dependent\")\n", - " self.dependence = dependence\n", - " independent_object, prob = random.choice(self.dependence)\n", - " self.attached = independent_object\n", - " self.location = self.attached.location\n", - "\n", - " def attach(self, independent_objects: list[str]) -> None:\n", - " \"\"\"Attach dependent object to independent object.\"\"\"\n", - " possible = []\n", - " for io in independent_objects:\n", - " for dep in self.dependence:\n", - " if dep[0].name == io:\n", - " possible.append(dep)\n", - "\n", - " independent_object, prob = random.choice(possible)\n", - "\n", - " if random.random() < prob:\n", - " self.attached = independent_object\n", - " else:\n", - " self.attached = None\n", - "\n", - " def move(self) -> None:\n", - " \"\"\"Move together with independent object.\"\"\"\n", - " if self.attached is not None:\n", - " self.location = self.attached.location\n", - "\n", - " def __repr__(self) -> str:\n", - " return f\"DependentObject({self.name}, {self.location}, {self.attached})\"\n", - "\n", - "\n", - "class Agent(Object):\n", - " def __init__(self, init_probs: dict) -> None:\n", - " super().__init__(\"agent\", \"agent\")\n", - " self.location = random.choices(\n", - " list(init_probs.keys()),\n", - " weights=list(init_probs.values()),\n", - " k=1,\n", - " )[0]\n", - "\n", - " def __repr__(self) -> str:\n", - " return f\"Agent({self.name}, {self.location})\"\n", - "\n", - " def move(self, location: str) -> None:\n", - " \"\"\"Agent can choose where to go.\"\"\"\n", - " self.location = location\n", - "\n", - "\n", - "class Rooms:\n", - " def __init__(\n", - " self,\n", - " rooms: list[str],\n", - " static_objects: list[StaticObject],\n", - " independent_objects: list[IndepdentObject],\n", - " dependent_objects: list[DependentObject],\n", - " agent: Agent,\n", - " seed: int,\n", - " ) -> None:\n", - " self.rooms = {room: [] for room in rooms}\n", - " self.static_objects = static_objects\n", - " self.independent_objects = independent_objects\n", - " self.dependent_objects = dependent_objects\n", - " self.agent = agent\n", - " self.seed = seed\n", - "\n", - " random.seed(seed)\n", - "\n", - " for obj in self.static_objects:\n", - " self.rooms[obj.location].append(obj)\n", - "\n", - " for obj in self.independent_objects:\n", - " self.rooms[obj.location].append(obj)\n", - "\n", - " for obj in self.dependent_objects:\n", - " self.rooms[obj.location].append(obj)\n", - "\n", - " self.rooms[agent.location].append(agent)\n", - "\n", - " def reset(self) -> tuple[list]:\n", - " \"\"\"Agent observes a sub graph and a question.\"\"\"\n", - " self.time = 0\n", - " return self.get_sub_graph(), self.get_question()\n", - "\n", - " def step(self, action: int) -> tuple[list]:\n", - " \"\"\"Agent takes an action to move to one of the rooms.\"\"\"\n", - " for room, val in self.rooms.items():\n", - " local_independent_objects = []\n", - " local_dependent_objects = []\n", - "\n", - " for val_ in val:\n", - " if isinstance(val_, IndepdentObject):\n", - " local_independent_objects.append(val_)\n", - " elif isinstance(val_, DependentObject):\n", - " local_dependent_objects.append(val_)\n", - "\n", - " if local_independent_objects and local_dependent_objects:\n", - " for dep in local_dependent_objects:\n", - " dep.attach([ind.name for ind in local_independent_objects])\n", - "\n", - " for ind in local_independent_objects:\n", - " ind.move()\n", - "\n", - " for dep in local_dependent_objects:\n", - " dep.move()\n", - "\n", - " self.agent.move(list(self.rooms.keys())[action])\n", - "\n", - " for obj in self.static_objects:\n", - " self.rooms[obj.location].append(obj)\n", - "\n", - " for obj in self.independent_objects:\n", - " self.rooms[obj.location].append(obj)\n", - "\n", - " for obj in self.dependent_objects:\n", - " self.rooms[obj.location].append(obj)\n", - "\n", - " self.rooms[self.agent.location].append(self.agent)\n", - "\n", - " self.hidden_state = self.get_hidden_state()\n", - "\n", - " self.time = +1\n", - "\n", - " return self.get_sub_graph(), self.get_question()\n", - "\n", - " def get_hidden_state(self) -> list:\n", - " self.hidden_state = []\n", - "\n", - " for room, val in self.rooms.items():\n", - " for val_ in val:\n", - " self.hidden_state.append([val_.name, \"atlocation\", room, self.time])\n", - "\n", - " return self.hidden_state\n", - "\n", - " def get_sub_graph(self) -> list:\n", - " self.sub_graph = []\n", - "\n", - " agent_room = None\n", - " for key, val in self.rooms.items():\n", - " for val_ in val:\n", - " if val_.name == \"agent\":\n", - " agent_room = key\n", - " if agent_room is not None:\n", - " break\n", - " if agent_room is not None:\n", - " break\n", - "\n", - " for obj in self.rooms[agent_room]:\n", - " self.sub_graph.append([obj.name, \"atlocation\", agent_room, self.time])\n", - "\n", - " return self.sub_graph\n", - "\n", - " def get_question(self) -> list:\n", - " self.hidden_state = self.get_hidden_state()\n", - "\n", - " while True:\n", - " self.question = random.choice(self.hidden_state)\n", - " if self.question[0] != \"agent\":\n", - " break\n", - "\n", - " return self.question\n", - "\n", - " def __repr__(self) -> str:\n", - " return f\"Rooms({self.rooms})\"" - ] - }, - { - "cell_type": "code", - "execution_count": 25, - "metadata": {}, - "outputs": [], - "source": [ - "# static objects\n", - "bed = StaticObject(\"bed\", \"bedroom\")\n", - "desk = StaticObject(\"desk\", \"officeroom\")\n", - "table = StaticObject(\"table\", \"livingroom\")\n", - "\n", - "# independent objects\n", - "tae = IndepdentObject(\"tae\", {\"officeroom\": 0.5, \"livingroom\": 0.5})\n", - "michael = IndepdentObject(\"michael\", {\"bedroom\": 0.5, \"livingroom\": 0.5})\n", - "vincent = IndepdentObject(\"vincent\", {\"bedroom\": 0.5, \"officeroom\": 0.5})\n", - "\n", - "# dependent objects\n", - "laptop = DependentObject(\"laptop\", [(tae, 0.7), (michael, 0.1), (vincent, 0.3)])\n", - "phone = DependentObject(\"phone\", [(tae, 0.3), (michael, 0.1), (vincent, 0.7)])\n", - "headset = DependentObject(\"headset\", [(tae, 0.5), (michael, 0.5), (vincent, 0.5)])\n", - "keyboard = DependentObject(\"keyboard\", [(tae, 0.9), (michael, 0.7), (vincent, 0.5)])\n", - "\n", - "# agent\n", - "agent = Agent({\"bedroom\": 0.333, \"officeroom\": 0.333, \"livingroom\": 0.333})\n", - "\n", - "# rooms\n", - "rooms = Rooms(\n", - " [\"bedroom\", \"officeroom\", \"livingroom\"],\n", - " [bed, desk, table],\n", - " [tae, michael, vincent],\n", - " [laptop, phone, headset, keyboard],\n", - " agent,\n", - " 42,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 31, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[['bed', 'atlocation', 'bedroom', 0],\n", - " ['vincent', 'atlocation', 'bedroom', 0],\n", - " ['phone', 'atlocation', 'bedroom', 0],\n", - " ['keyboard', 'atlocation', 'bedroom', 0],\n", - " ['bed', 'atlocation', 'bedroom', 0],\n", - " ['phone', 'atlocation', 'bedroom', 0],\n", - " ['agent', 'atlocation', 'bedroom', 0],\n", - " ['desk', 'atlocation', 'officeroom', 0],\n", - " ['tae', 'atlocation', 'officeroom', 0],\n", - " ['laptop', 'atlocation', 'officeroom', 0],\n", - " ['headset', 'atlocation', 'officeroom', 0],\n", - " ['desk', 'atlocation', 'officeroom', 0],\n", - " ['tae', 'atlocation', 'officeroom', 0],\n", - " ['vincent', 'atlocation', 'officeroom', 0],\n", - " ['laptop', 'atlocation', 'officeroom', 0],\n", - " ['headset', 'atlocation', 'officeroom', 0],\n", - " ['keyboard', 'atlocation', 'officeroom', 0],\n", - " ['table', 'atlocation', 'livingroom', 0],\n", - " ['michael', 'atlocation', 'livingroom', 0],\n", - " ['agent', 'atlocation', 'livingroom', 0],\n", - " ['table', 'atlocation', 'livingroom', 0],\n", - " ['michael', 'atlocation', 'livingroom', 0]]" - ] - }, - "execution_count": 31, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "rooms.get_hidden_state()" - ] - }, - { - "cell_type": "code", - "execution_count": 28, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "([['bed', 'atlocation', 'bedroom', 1],\n", - " ['vincent', 'atlocation', 'bedroom', 1],\n", - " ['phone', 'atlocation', 'bedroom', 1],\n", - " ['keyboard', 'atlocation', 'bedroom', 1],\n", - " ['bed', 'atlocation', 'bedroom', 1],\n", - " ['phone', 'atlocation', 'bedroom', 1],\n", - " ['agent', 'atlocation', 'bedroom', 1]],\n", - " ['bed', 'atlocation', 'bedroom', 1])" - ] - }, - "execution_count": 28, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "rooms.step(0)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# static objects\n", - "bed = StaticObject(\"bed\", \"bedroom\")\n", - "desk = StaticObject(\"desk\", \"officeroom\")\n", - "table = StaticObject(\"table\", \"livingroom\")\n", - "\n", - "# independent objects\n", - "tae = IndepdentObject(\"tae\", {\"officeroom\": 0.5, \"livingroom\": 0.5})\n", - "michael = IndepdentObject(\"michael\", {\"bedroom\": 0.5, \"livingroom\": 0.5})\n", - "vincent = IndepdentObject(\"vincent\", {\"bedroom\": 0.5, \"officeroom\": 0.5})\n", - "\n", - "# dependent objects\n", - "laptop = DependentObject(\"laptop\", [(tae, 0.7), (michael, 0.1), (vincent, 0.3)])\n", - "phone = DependentObject(\"phone\", [(tae, 0.3), (michael, 0.1), (vincent, 0.7)])\n", - "\n", - "# agent\n", - "agent = Agent({\"bedroom\": 0.333, \"officeroom\": 0.333, \"livingroom\": 0.333})\n", - "\n", - "# rooms\n", - "rooms = Rooms(\n", - " [\"bedroom\", \"officeroom\", \"livingroom\"],\n", - " [bed, desk, table],\n", - " [tae, michael, vincent],\n", - " [laptop, phone],\n", - " agent,\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 263, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Rooms({'bedroom': [StaticObject(bed, bedroom), IndependentObject(vincent, bedroom), DependentObject(phone, bedroom, IndependentObject(vincent, bedroom)), StaticObject(bed, bedroom), IndependentObject(vincent, bedroom), DependentObject(phone, bedroom, IndependentObject(vincent, bedroom)), Agent(agent, bedroom), StaticObject(bed, bedroom), IndependentObject(michael, bedroom), IndependentObject(vincent, bedroom), DependentObject(phone, bedroom, IndependentObject(vincent, bedroom)), StaticObject(bed, bedroom), IndependentObject(michael, bedroom), DependentObject(phone, bedroom, IndependentObject(vincent, bedroom)), StaticObject(bed, bedroom), IndependentObject(michael, bedroom), IndependentObject(vincent, bedroom), DependentObject(phone, bedroom, IndependentObject(vincent, bedroom)), Agent(agent, bedroom)], 'officeroom': [StaticObject(desk, officeroom), Agent(agent, bedroom), StaticObject(desk, officeroom), StaticObject(desk, officeroom), Agent(agent, bedroom), StaticObject(desk, officeroom), IndependentObject(tae, livingroom), IndependentObject(vincent, bedroom), StaticObject(desk, officeroom)], 'livingroom': [StaticObject(table, livingroom), IndependentObject(tae, livingroom), IndependentObject(michael, bedroom), DependentObject(laptop, livingroom, None), StaticObject(table, livingroom), IndependentObject(tae, livingroom), IndependentObject(michael, bedroom), DependentObject(laptop, livingroom, None), StaticObject(table, livingroom), IndependentObject(tae, livingroom), DependentObject(laptop, livingroom, None), StaticObject(table, livingroom), DependentObject(laptop, livingroom, None), Agent(agent, bedroom), StaticObject(table, livingroom), IndependentObject(tae, livingroom), DependentObject(laptop, livingroom, None)]})" - ] - }, - "execution_count": 263, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "rooms" - ] - }, - { - "cell_type": "code", - "execution_count": 254, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[['bed', 'atlocation', 'bedroom'], ['vincent', 'atlocation', 'bedroom'], ['phone', 'atlocation', 'bedroom'], ['desk', 'atlocation', 'officeroom'], ['agent', 'atlocation', 'officeroom'], ['table', 'atlocation', 'livingroom'], ['tae', 'atlocation', 'livingroom'], ['michael', 'atlocation', 'livingroom'], ['laptop', 'atlocation', 'livingroom']]" - ] - }, - "execution_count": 254, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "rooms.get_hidden_state()" - ] - }, - { - "cell_type": "code", - "execution_count": 255, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[['desk', 'atlocation', 'officeroom'], ['agent', 'atlocation', 'officeroom']]" - ] - }, - "execution_count": 255, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "rooms.get_sub_graph()" - ] - }, - { - "cell_type": "code", - "execution_count": 256, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "([['desk', 'atlocation', 'officeroom'], ['agent', 'atlocation', 'officeroom']], ['bed', 'atlocation', 'bedroom'])" - ] - }, - "execution_count": 256, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "rooms.reset()" - ] - }, - { - "cell_type": "code", - "execution_count": 262, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "([['bed', 'atlocation', 'bedroom'], ['vincent', 'atlocation', 'bedroom'], ['phone', 'atlocation', 'bedroom'], ['bed', 'atlocation', 'bedroom'], ['vincent', 'atlocation', 'bedroom'], ['phone', 'atlocation', 'bedroom'], ['agent', 'atlocation', 'bedroom'], ['bed', 'atlocation', 'bedroom'], ['michael', 'atlocation', 'bedroom'], ['vincent', 'atlocation', 'bedroom'], ['phone', 'atlocation', 'bedroom'], ['bed', 'atlocation', 'bedroom'], ['michael', 'atlocation', 'bedroom'], ['phone', 'atlocation', 'bedroom'], ['bed', 'atlocation', 'bedroom'], ['michael', 'atlocation', 'bedroom'], ['vincent', 'atlocation', 'bedroom'], ['phone', 'atlocation', 'bedroom'], ['agent', 'atlocation', 'bedroom']], ['phone', 'atlocation', 'bedroom'])" - ] - }, - "execution_count": 262, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "rooms.step(0)" - ] - }, - { - "cell_type": "code", - "execution_count": 240, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "['michael', 'atlocation', 'bedroom']" - ] - }, - "execution_count": 240, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "rooms.get_question()" - ] - }, - { - "cell_type": "code", - "execution_count": 192, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "DependentObject(laptop, officeroom, IndependentObject(tae, officeroom))" - ] - }, - "execution_count": 192, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "rooms.dependent_objects[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 150, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "IndependentObject(michael, independent, livingroom)\n" - ] - } - ], - "source": [ - "laptop = DependentObject(\"laptop\", [(tae, 0.7), (michael, 0.1), (vincent, 0.3)])\n", - "print(laptop.attached)" - ] - }, - { - "cell_type": "code", - "execution_count": 165, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "IndependentObject(tae, independent, officeroom)\n" - ] - } - ], - "source": [ - "laptop.attach([\"tae\"])\n", - "print(laptop.attached)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 131, - "metadata": {}, - "outputs": [ - { - "ename": "TypeError", - "evalue": "list indices must be integers or slices, not str", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[131], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m laptop\u001b[39m.\u001b[39;49mattach([\u001b[39m\"\u001b[39;49m\u001b[39mMichael\u001b[39;49m\u001b[39m\"\u001b[39;49m])\n", - "Cell \u001b[0;32mIn[125], line 73\u001b[0m, in \u001b[0;36mDependentObject.attach\u001b[0;34m(self, independent_objects)\u001b[0m\n\u001b[1;32m 70\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Attach dependent object to independent object.\"\"\"\u001b[39;00m\n\u001b[1;32m 71\u001b[0m independent_object \u001b[39m=\u001b[39m random\u001b[39m.\u001b[39mchoice(independent_objects)\n\u001b[0;32m---> 73\u001b[0m \u001b[39mif\u001b[39;00m random\u001b[39m.\u001b[39mrandom() \u001b[39m<\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mdependence[independent_object]:\n\u001b[1;32m 74\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mattached \u001b[39m=\u001b[39m independent_object\n\u001b[1;32m 75\u001b[0m \u001b[39melse\u001b[39;00m:\n", - "\u001b[0;31mTypeError\u001b[0m: list indices must be integers or slices, not str" - ] - } - ], - "source": [ - "laptop.attach()" - ] - }, - { - "cell_type": "code", - "execution_count": 91, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.5958237632388955" - ] - }, - "execution_count": 91, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "random.random()" - ] - }, - { - "cell_type": "code", - "execution_count": 87, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(IndependentObject(vincent, independent, bedroom), 0.3)" - ] - }, - "execution_count": 87, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "random.choice(laptop.dependence)" - ] - }, - { - "cell_type": "code", - "execution_count": 80, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "DependentObject(laptop, dependent, [(IndependentObject(tae, independent, officeroom), 0.7), (IndependentObject(michael, independent, livingroom), 0.1), (IndependentObject(vincent, independent, bedroom), 0.3)])" - ] - }, - "execution_count": 80, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "laptop" - ] - }, - { - "cell_type": "code", - "execution_count": 82, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[(IndependentObject(tae, independent, officeroom), 0.7),\n", - " (IndependentObject(michael, independent, livingroom), 0.1),\n", - " (IndependentObject(vincent, independent, bedroom), 0.3)]" - ] - }, - "execution_count": 82, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "laptop.dependence" - ] - }, - { - "cell_type": "code", - "execution_count": 68, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'officeroom'" - ] - }, - "execution_count": 68, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "agent.move(\"officeroom\")\n", - "agent.location" - ] - }, - { - "cell_type": "code", - "execution_count": 38, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "IndependentObject(Tae, independent, officeroom)" - ] - }, - "execution_count": 38, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "tae.move()\n", - "tae" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[Object(bed, static, Rooms({'bedroom': [Object(bed, static, Rooms({...}))], 'officeroom': [], 'livingroom': []}))]" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "bed.rooms.rooms[\"bedroom\"][0].rooms.rooms[\"bedroom\"]" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "('bedroom', None)" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import random\n", - "\n", - "\n", - "class Object:\n", - " def __init__(\n", - " self,\n", - " name: str,\n", - " type: str,\n", - " probs: dict,\n", - " rooms: list = [\"bedroom\", \"officeroom\", \"livingroom\"],\n", - " ) -> None:\n", - " \"\"\"Entity, e.g. human, object, room.\n", - "\n", - " Args\n", - " ----\n", - " name: e.g., Tae, laptop, bed\n", - " type: e.g., static, independent, dependent\n", - " probs:\n", - "\n", - " if type == \"static\", e.g., {\"officeroom\": 0, \"livingroom\": 0, \"bedroom\": 1}\n", - " elif type == \"independent\", e.g., {\"bedroom\": 0.5, \"livingroom\": 0.5}\n", - " elif type == \"dependent\", e.g., {\"Tae\": 0.7, \"Michael\": 0.1, \"Vincent\": 0.3}\n", - " elif type == \"agent\", e.g., {\"officeroom\": 0.333, \"livingroom\": 0.333,\n", - " \"bedroom\": 0.333}\n", - "\n", - " \"\"\"\n", - " self.name = name\n", - " self.type = type\n", - " assert self.type in [\"static\", \"independent\", \"dependent\", \"agent\"]\n", - " self.probs = probs\n", - " self.rooms = rooms\n", - " self.move()\n", - "\n", - " def move(self) -> None:\n", - " if self.type == \"static\":\n", - " pass\n", - "\n", - " self.loc = random.choices(\n", - " list(self.probs.keys()),\n", - " weights=list(self.probs.values()),\n", - " k=1,\n", - " )[0]\n", - "\n", - " def __repr__(self) -> str:\n", - " return f\"Entity(name: {self.name}, type: {self.type}, location: {self.loc})\"\n", - "\n", - " def change_location(self, loc: str) -> None:\n", - " self.loc = loc\n", - "\n", - "\n", - "foo = Object(\"foo\", \"static\", {\"bedroom\": 1.0, \"livingroom\": 0.2})\n", - "foo.loc, foo.dependent_probs" - ] - }, - { - "cell_type": "code", - "execution_count": 77, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "observation: [('table', 'atlocation', 'livingroom', 0), ('laptop', 'atlocation', 'livingroom', 0), ('phone', 'atlocation', 'livingroom', 0), ('agent', 'atlocation', 'livingroom', 0)]\n", - "\n", - "question: ('bed', 'atlocation', 'bedroom')\n" - ] - } - ], - "source": [ - "import random\n", - "from pprint import pprint\n", - "\n", - "\n", - "class Pomdp:\n", - " def __init__(\n", - " self, rooms: list = [\"bedroom\", \"officeroom\", \"livingroom\"], seed: int = 42\n", - " ) -> None:\n", - " random.seed(seed)\n", - " self.rooms = rooms\n", - "\n", - " def reset(self):\n", - " self.objects = []\n", - "\n", - " # add static objects\n", - " self.objects.append(\n", - " Object(\n", - " \"bed\",\n", - " \"static\",\n", - " {\"officeroom\": 0, \"livingroom\": 0, \"bedroom\": 1},\n", - " )\n", - " )\n", - " self.objects.append(\n", - " Object(\n", - " \"desk\",\n", - " \"static\",\n", - " {\"officeroom\": 1, \"livingroom\": 0, \"bedroom\": 0},\n", - " )\n", - " )\n", - " self.objects.append(\n", - " Object(\n", - " \"table\",\n", - " \"static\",\n", - " {\"officeroom\": 0, \"livingroom\": 1, \"bedroom\": 0},\n", - " )\n", - " )\n", - "\n", - " # add independent objects\n", - " self.objects.append(\n", - " Object(\n", - " \"Tae\",\n", - " \"independent\",\n", - " {\"officeroom\": 0.5, \"livingroom\": 0.5, \"bedroom\": 0},\n", - " )\n", - " )\n", - " self.objects.append(\n", - " Object(\n", - " \"Michael\",\n", - " \"independent\",\n", - " {\"officeroom\": 0, \"livingroom\": 0.5, \"bedroom\": 0.5},\n", - " )\n", - " )\n", - " self.objects.append(\n", - " Object(\n", - " \"Vincent\",\n", - " \"independent\",\n", - " {\"officeroom\": 0.5, \"livingroom\": 0, \"bedroom\": 0.5},\n", - " )\n", - " )\n", - "\n", - " # add dependent objects\n", - " self.objects.append(\n", - " Object(\n", - " \"laptop\",\n", - " \"dependent\",\n", - " {\"officeroom\": 0.5, \"livingroom\": 0.5, \"bedroom\": 0},\n", - " {\"Tae\": 0.7, \"Michael\": 0.1, \"Vincent\": 0.3},\n", - " )\n", - " )\n", - " self.objects.append(\n", - " Object(\n", - " \"phone\",\n", - " \"dependent\",\n", - " {\"officeroom\": 0, \"livingroom\": 0.5, \"bedroom\": 0.5},\n", - " {\"Tae\": 0.3, \"Michael\": 0.1, \"Vincent\": 0.7},\n", - " )\n", - " )\n", - "\n", - " # add agent\n", - " self.objects.append(\n", - " Object(\n", - " \"agent\",\n", - " \"agent\",\n", - " {\"officeroom\": 0.333, \"livingroom\": 0.333, \"bedroom\": 0.333},\n", - " )\n", - " )\n", - "\n", - " # get basic info\n", - " self.num_rooms = len(self.rooms)\n", - "\n", - " self.num_static_objects = sum(\n", - " [1 for obj in self.objects if obj.type == \"static\"]\n", - " )\n", - " self.num_independent_objects = sum(\n", - " [1 for obj in self.objects if obj.type == \"independent\"]\n", - " )\n", - " self.num_dependent_objects = sum(\n", - " [1 for obj in self.objects if obj.type == \"dependent\"]\n", - " )\n", - "\n", - " self.num_hidden_states = self.num_rooms ** (\n", - " self.num_independent_objects + self.num_dependent_objects + 1\n", - " )\n", - " self.num_observations = (\n", - " 2 ** (self.num_independent_objects + self.num_dependent_objects)\n", - " * self.num_rooms\n", - " )\n", - " self.time = 0\n", - "\n", - " return self._get_observation()\n", - "\n", - " def _compute_hidden_state(self):\n", - " if self.time == 0:\n", - " self.hidden_state_room_perspective = {room: [] for room in self.rooms}\n", - " for obj in self.objects:\n", - " self.hidden_state_room_perspective[obj.loc].append(obj)\n", - "\n", - " self.hidden_state_kg_perspective = [\n", - " (obj.name, \"atlocation\", obj.loc)\n", - " for room, objects in self.hidden_state_room_perspective.items()\n", - " for obj in objects\n", - " ]\n", - " else:\n", - " for room, objects in self.hidden_state_room_perspective.items():\n", - " independent_objects = [\n", - " obj for obj in objects if obj.type == \"independent\"\n", - " ]\n", - " dependent_objects = [obj for obj in objects if obj.type == \"dependent\"]\n", - "\n", - " io_do_pairs = []\n", - " for do in dependent_objects:\n", - " matches =[]\n", - " for io in independent_objects:\n", - " if io.name in list(do.dependent_probs.keys()):\n", - " matches.append(io)\n", - " if len(matches) > 0:\n", - " match = random.choice(matches)\n", - " if random.random() < do.dependent_probs[match.name]:\n", - " io_do_pairs.append((match, do))\n", - " \n", - " for io in independent_objects:\n", - " if random.random() < io.probs[room]:\n", - " \n", - "\n", - "\n", - " \n", - " for obj in objects:\n", - " if obj.type == \"static\":\n", - " continue\n", - "\n", - " if obj.type == \"independent\":\n", - " obj.change_location(\n", - " random.choices(\n", - " list(obj.probs.keys()),\n", - " weights=list(obj.probs.values()),\n", - " k=1,\n", - " )[0]\n", - " )\n", - " elif obj.type == \"dependent\":\n", - " if (\n", - " random.random()\n", - " < obj.dependent_probs[\n", - " self.hidden_state_room_perspective[obj.loc][0].name\n", - " ]\n", - " ):\n", - " obj.change_location(\n", - " random.choices(\n", - " list(obj.probs.keys()),\n", - " weights=list(obj.probs.values()),\n", - " k=1,\n", - " )[0]\n", - " )\n", - "\n", - " def _get_observation(self):\n", - " self._compute_hidden_state()\n", - " for obj in self.objects:\n", - " if obj.name == \"agent\":\n", - " self.agent_location = obj.loc\n", - "\n", - " self.sub_graph_room_perspective = [\n", - " obj for obj in self.objects if obj.loc == self.agent_location\n", - " ]\n", - " self.sub_graph_kg_perspective = [\n", - " (obj.name, \"atlocation\", obj.loc, self.time)\n", - " for obj in self.sub_graph_room_perspective\n", - " ]\n", - " self.question = random.choice(self.hidden_state_kg_perspective)\n", - "\n", - " return self.sub_graph_kg_perspective, self.question\n", - "\n", - " def step(self):\n", - " self.time += 1\n", - " self._get_observation()\n", - "\n", - "\n", - "pomdp = Pomdp(seed=42)\n", - "observation, question = pomdp.reset()\n", - "print(f\"observation: {observation}\")\n", - "print()\n", - "print(f\"question: {question}\")\n", - "\n", - "# pprint(f\"objects: {pomdp.objects}\")\n", - "# print()\n", - "# pprint(f\"hidden_state_room_perspective: {pomdp.hidden_state_room_perspective}\")\n", - "# print()\n", - "# pprint(f\"hidden_state_kg_perspective: {pomdp.hidden_state_kg_perspective}\")\n", - "# print()\n", - "# pprint(f\"sub_graph_room_perspective: {pomdp.sub_graph_room_perspective}\")\n", - "# print()\n", - "# pprint(f\"sub_graph_kg_perspective: {pomdp.sub_graph_kg_perspective}\")\n", - "# print()\n", - "# pprint(f\"question: {pomdp.question}\\n\")\n", - "# print()\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0.4183346213273226" - ] - }, - "execution_count": 1, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import random\n", - "\n", - "random.random()" - ] - }, - { - "cell_type": "code", - "execution_count": 78, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'bedroom': [Entity(name: bed, type: static, location: bedroom),\n", - " Entity(name: Michael, type: independent, location: bedroom),\n", - " Entity(name: Vincent, type: independent, location: bedroom)],\n", - " 'officeroom': [Entity(name: desk, type: static, location: officeroom),\n", - " Entity(name: laptop, type: dependent, location: officeroom),\n", - " Entity(name: agent, type: agent, location: officeroom)],\n", - " 'livingroom': [Entity(name: table, type: static, location: livingroom),\n", - " Entity(name: Tae, type: independent, location: livingroom),\n", - " Entity(name: phone, type: dependent, location: livingroom)]}" - ] - }, - "execution_count": 78, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "observation, question = pomdp.reset()\n", - "pomdp.hidden_state_room_perspective" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": {}, - "outputs": [], - "source": [ - "pomdp.objects[0].change_location(\"livingroom\")" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Entity(name: bed, type: static, location: livingroom)" - ] - }, - "execution_count": 63, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pomdp.objects[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": {}, - "outputs": [], - "source": [ - "pomdp.objects[0].change_location(\"officeroom\")" - ] - }, - { - "cell_type": "code", - "execution_count": 65, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "Entity(name: bed, type: static, location: officeroom)" - ] - }, - "execution_count": 65, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pomdp.objects[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pomdp.time" - ] - }, - { - "cell_type": "code", - "execution_count": 50, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "<__main__.Entity at 0x7fbd1ee355b0>" - ] - }, - "execution_count": 50, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "pomdp.entities[0]" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "ename": "TypeError", - "evalue": "sample() missing 1 required positional argument: 'k'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mTypeError\u001b[0m Traceback (most recent call last)", - "Cell \u001b[0;32mIn[4], line 2\u001b[0m\n\u001b[1;32m 1\u001b[0m baz \u001b[39m=\u001b[39m {\u001b[39m\"\u001b[39m\u001b[39mfoo\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m0.2\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mbar\u001b[39m\u001b[39m\"\u001b[39m: \u001b[39m0.8\u001b[39m}\n\u001b[0;32m----> 2\u001b[0m random\u001b[39m.\u001b[39;49msample(baz)\n", - "\u001b[0;31mTypeError\u001b[0m: sample() missing 1 required positional argument: 'k'" - ] - } - ], - "source": [ - "baz = {\"foo\": 0.2, \"bar\": 0.8}\n", - "random.sample(baz)" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[0;31mSignature:\u001b[0m \u001b[0mrandom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchoice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mseq\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mDocstring:\u001b[0m Choose a random element from a non-empty sequence.\n", - "\u001b[0;31mFile:\u001b[0m /usr/lib/python3.9/random.py\n", - "\u001b[0;31mType:\u001b[0m method" - ] - } - ], - "source": [ - "?random.choice" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'bar'" - ] - }, - "execution_count": 21, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "random.choices(list(baz.keys()), weights=list(baz.values()), k=1)[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import random\n", - "\n", - "\n", - "class Pomdp:\n", - " def __init__(\n", - " self,\n", - " humans: dict,\n", - " entities: dict,\n", - " world_knowledge: list,\n", - " init_probs: list,\n", - " seed: int = 42,\n", - " ) -> None:\n", - " \"\"\"POMDP\"\"\"\n", - " self.seed = seed\n", - " random.seed(self.seed)\n", - " self.humans = humans\n", - " self.objects = entities\n", - " self.world_knowledge = world_knowledge\n", - "\n", - " self.static_objects = [\n", - " entity\n", - " for entity, characteristic in entities.items()\n", - " if characteristic[\"type\"] == \"static_object\"\n", - " ]\n", - " self.non_static_objects = [\n", - " entity\n", - " for entity, characteristic in entities.items()\n", - " if characteristic[\"type\"] == \"non_static_object\"\n", - " ]\n", - " self.rooms = [\n", - " entity\n", - " for entity, characteristic in entities.items()\n", - " if characteristic[\"type\"] == \"room\"\n", - " ]\n", - "\n", - " self.objects = self.static_objects + self.non_static_objects\n", - " self.world_knowledge = world_knowledge\n", - " self.init_probs = init_probs\n", - "\n", - " self.num_humans = len(self.humans) # number of humans\n", - " self.num_static_objects = len(self.static_objects) # number of static objects\n", - " self.num_non_static_objects = len(\n", - " self.non_static_objects\n", - " ) # number of static objects\n", - " self.num_rooms = len(self.rooms) # number of rooms\n", - "\n", - " self.num_hidden_states = self.num_rooms ** (\n", - " self.num_humans + self.num_non_static_objects + 1\n", - " )\n", - " self.num_observations = (\n", - " 2 ** (self.num_humans + self.num_non_static_objects) * self.num_rooms\n", - " )\n", - " self._init_environment()\n", - "\n", - " def _init_environment(self) -> None:\n", - " \"\"\"Populate the rooms with objects and humans at uniformly random.\"\"\"\n", - " self.env = {\n", - " \"bedroom\": {\"static_objects\": []},\n", - " \"non_static_objects\": [],\n", - " \"humans\": [],\n", - " }\n", - " self.env[\"bedroom\"] = {\"static_object\": [\"bed\"]}\n", - " self.env[\"officeroom\"] = {\"static_object\": [\"desk\"]}\n", - " self.env[\"livingroom\"] = {\"static_object\": [\"table\"]}\n", - "\n", - " for human in self.humans:\n", - " room = random.choice([\"bedroom\", \"officeroom\", \"livingroom\"])\n", - " self.env[room].append(human)\n", - "\n", - " for obj in self.non_static_objects:\n", - " room = random.choice([\"bedroom\", \"officeroom\", \"livingroom\"])\n", - " self.env[room].append(obj)\n", - "\n", - " room = random.choice([\"bedroom\", \"officeroom\", \"livingroom\"])\n", - " self.env[room].append(\"agent\")\n", - "\n", - " def step(self, action: int) -> tuple:\n", - " \"\"\"Take a step in the environment.\"\"\"\n", - " self.env[\"agent\"] = (\"atlocation\", self.rooms[action])\n", - "\n", - "\n", - "config = {\n", - " \"seed\": 42,\n", - " \"humans\": {\n", - " \"Tae\": {\n", - " \"room\": {\"bedroom\": 0.7, \"officeroom\": 0.1, \"livingroom\": 0.2},\n", - " \"object\": {\n", - " \"laptop\": 0.3,\n", - " \"phone\": 0.3,\n", - " },\n", - " },\n", - " \"Michael\": {\n", - " \"room\": {\"bedroom\": 0.2, \"officeroom\": 0.7, \"livingroom\": 0.1},\n", - " \"object\": {\n", - " \"laptop\": 0.9,\n", - " },\n", - " },\n", - " \"Vincent\": {\n", - " \"room\": {\"bedroom\": 0.1, \"officeroom\": 0.2, \"livingroom\": 0.7},\n", - " \"object\": {\n", - " \"phone\": 0.8,\n", - " },\n", - " },\n", - " },\n", - " \"entities\": {\n", - " \"bedroom\": {\"type\": \"room\"},\n", - " \"officeroom\": {\"type\": \"room\"},\n", - " \"livingroom\": {\"type\": \"room\"},\n", - " \"bed\": {\"type\": \"static_object\"},\n", - " \"desk\": {\"type\": \"static_object\"},\n", - " \"table\": {\"type\": \"static_object\"},\n", - " \"laptop\": {\"type\": \"non_static_object\"},\n", - " \"phone\": {\"type\": \"non_static_object\"},\n", - " \"Tae\": {\"type\": \"human\"},\n", - " \"Michael\": {\"type\": \"human\"},\n", - " \"Vincent\": {\"type\": \"human\"},\n", - " \"agent\": {\"type\": \"agent\"},\n", - " },\n", - " \"world_knowledge\": [\n", - " (\"static_object\", \"PartOf\", \"room\"),\n", - " (\"non_static_object\", \"atlocation\", \"static_object\"),\n", - " (\"human\", \"atlocation\", \"room\"),\n", - " (\"agent\", \"atlocation\", \"room\"),\n", - " (\"human\", \"Owns\", \"non_static_object\"),\n", - " ],\n", - " \"init_probs\": [\n", - " {\"bed\": [(\"PartOf\", \"bedroom\", 1.0)]},\n", - " {\"desk\": [(\"PartOf\", \"officeroom\", 1.0)]},\n", - " {\"bed\": [(\"PartOf\", \"livingroom\", 1.0)]},\n", - " {\"laptop\": [(\"atlocation\", \"desk\", 0.7), (\"atlocation\", \"bed\", 0.3)]},\n", - " {\"phone\": [(\"atlocation\", \"table\", 0.7), (\"atlocation\", \"bed\", 0.3)]},\n", - " {\n", - " \"Tae\": [\n", - " (\"atlocation\", \"bedroom\", 0.7),\n", - " (\"atlocation\", \"officeroom\", 0.1),\n", - " (\"atlocation\", \"livingroom\", 0.2),\n", - " ]\n", - " },\n", - " {\n", - " \"Michael\": [\n", - " (\"atlocation\", \"bedroom\", 0.2),\n", - " (\"atlocation\", \"officeroom\", 0.7),\n", - " (\"atlocation\", \"livingroom\", 0.1),\n", - " ]\n", - " },\n", - " {\n", - " \"Vincent\": [\n", - " (\"atlocation\", \"bedroom\", 0.1),\n", - " (\"atlocation\", \"officeroom\", 0.2),\n", - " (\"atlocation\", \"livingroom\", 0.7),\n", - " ]\n", - " },\n", - " {\n", - " \"agent\": [\n", - " (\"atlocation\", \"bedroom\", 0.333),\n", - " (\"atlocation\", \"officeroom\", 0.333),\n", - " (\"atlocation\", \"livingroom\", 0.333),\n", - " ]\n", - " },\n", - " ],\n", - "}\n", - "\n", - "pomdp = Pomdp(**config)\n", - "pomdp.humans, pomdp.objects, pomdp.rooms" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "class Entity:\n", - " def __init__(self, name: str, kind: str) -> None:\n", - " \"\"\"Entity\"\"\"\n", - " self.name = name\n", - " self.kind" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pomdp.env" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pomdp.step(1)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pomdp.env" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "random.choice([\"bedroom\", \"officeroom\", \"livingroom\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pomdp.humans.keys(), pomdp.rooms" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pomdp.hidden_states" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pomdp.num_humans, pomdp.num_static_objects, pomdp.num_non_static_objects, pomdp.num_rooms, pomdp.num_hidden_states, pomdp.num_observations" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "pomdp.num_observations, pomdp.num_hidden_states" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "list(config[\"objects\"].keys())" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "len(config[\"objects\"])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "w = np.array(\n", - " [\n", - " [0, 0.4, 0.8, 0],\n", - " [1, 0, 0, 0.7],\n", - " [0, 0, 0, 0.3],\n", - " [0, 0.6, 0.2, 0],\n", - " ]\n", - ")\n", - "w_ = np.eye(4)\n", - "\n", - "foo = []\n", - "for i in range(1000):\n", - " w_ = w @ w_\n", - " print(w_)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "alpha = 0.3\n", - "beta = 0.7\n", - "\n", - "w = np.array(\n", - " [\n", - " [1 - beta, beta, 0, alpha * beta],\n", - " [(1 - alpha) * beta, 1 - beta, 0, 0],\n", - " [0, 0, 1 - beta, (1 - alpha) * beta],\n", - " [alpha * beta, 0, beta, 1 - beta],\n", - " ]\n", - ")\n", - "w_ = np.eye(4)\n", - "\n", - "foo = []\n", - "for i in range(1000):\n", - " w_ = w @ w_\n", - " print(w_)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "alpha = 0.3\n", - "beta = 0.7\n", - "\n", - "w = np.array(\n", - " [\n", - " [0.3, 0.7, 0, 0],\n", - " [0.2, 0.3, 0, 0],\n", - " [0, 0, 0.3, 0.5],\n", - " [0.5, 0, 0.7, 0.5],\n", - " ]\n", - ")\n", - "w_ = np.eye(4)\n", - "\n", - "foo = []\n", - "for i in range(1000):\n", - " w_ = w @ w_\n", - " print(w_)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "w = np.array(\n", - " [\n", - " [0, 0, 1],\n", - " [1, 0, 0],\n", - " [0, 1, 0],\n", - " ]\n", - ")\n", - "w_ = np.eye(3)\n", - "\n", - "foo = []\n", - "for i in range(1000):\n", - " w_ = w @ w_\n", - " print(w_)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "alpha = 0.3\n", - "beta = 0.7\n", - "\n", - "w = np.array(\n", - " [\n", - " [1 - alpha, alpha * (1 - beta), 0, alpha * beta],\n", - " [alpha * (1 - beta), 1 - alpha, 0, 0],\n", - " [0, 0, 1 - alpha, alpha * (1 - beta)],\n", - " [alpha * beta, 0, alpha * (1 - beta), 1 - alpha],\n", - " ]\n", - ")\n", - "w_ = np.eye(4)\n", - "\n", - "foo = []\n", - "for i in range(1000):\n", - " w_ = w @ w_\n", - " print(w_)\n", - " foo.append(w_[0, 0])" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "np.random.randint(0, 10, 5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "\n", - "w = np.array(\n", - " [\n", - " [0, 0.2, 0],\n", - " [1, 0, 1],\n", - " [0, 0.8, 0],\n", - " ]\n", - ")\n", - "x = np.array([1 / 3, 1 / 3, 1 / 3])\n", - "\n", - "for i in range(100):\n", - " x = w @ x\n", - " print(x)\n", - " print()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "np.random.randint(0, 10, 5)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "w = np.random.randint(0, 10, (5, 5))\n", - "w = w / w.sum(axis=1)[:, np.newaxis]\n", - "\n", - "x = np.random.randint(0, 10, 5)\n", - "x = x / x.sum()\n", - "# foo, foo.sum(axis=1)\n", - "\n", - "for i in range(100):\n", - " x = w @ x\n", - " print(x)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "x" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "w" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "foo.sum(axis=1)[:, np.newaxis]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy as np\n", - "\n", - "\n", - "w = np.array(\n", - " [\n", - " [3 / 4, 1 / 4, 0, 0, 0],\n", - " [3 / 4, 0, 1 / 4, 0, 0],\n", - " [3 / 4, 0, 0, 1 / 4, 0],\n", - " [3 / 4, 0, 0, 0, 1 / 4],\n", - " [1, 0, 0, 0, 0],\n", - " ]\n", - ")\n", - "w_ = np.eye(5)\n", - "\n", - "for i in range(100):\n", - " w_ = np.dot(w, w_)\n", - " print(w_)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "w.shape, x.shape" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "human-memory", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.17" - }, - "orig_nbformat": 4 - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/room-env-v2.ipynb b/room-env-v2.ipynb index ad922b4..91ee1a2 100644 --- a/room-env-v2.ipynb +++ b/room-env-v2.ipynb @@ -540,33 +540,14 @@ }, { "cell_type": "code", - "execution_count": 72, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/tk/.virtualenvs/humemai/lib/python3.10/site-packages/gymnasium/utils/passive_env_checker.py:168: DeprecationWarning: \u001b[33mWARN: Current gymnasium version requires that `Env.reset` can be passed a `seed` instead of using `Env.seed` for resetting the environment random number generator.\u001b[0m\n", - " logger.deprecation(\n", - "/home/tk/.virtualenvs/humemai/lib/python3.10/site-packages/gymnasium/utils/passive_env_checker.py:181: DeprecationWarning: \u001b[33mWARN: Current gymnasium version requires that `Env.reset` can be passed `options` to allow the environment initialisation to be passed additional information.\u001b[0m\n", - " logger.deprecation(\n", - "/home/tk/.virtualenvs/humemai/lib/python3.10/site-packages/gymnasium/utils/passive_env_checker.py:127: UserWarning: \u001b[33mWARN: The obs returned by the `reset()` method should be an int or np.int64, actual type: \u001b[0m\n", - " logger.warn(f\"{pre} should be an int or np.int64, actual type: {type(obs)}\")\n", - "/home/tk/.virtualenvs/humemai/lib/python3.10/site-packages/gymnasium/utils/passive_env_checker.py:159: UserWarning: \u001b[33mWARN: The obs returned by the `reset()` method is not within the observation space.\u001b[0m\n", - " logger.warn(f\"{pre} is not within the observation space.\")\n", - "/home/tk/.virtualenvs/humemai/lib/python3.10/site-packages/gymnasium/utils/passive_env_checker.py:127: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method should be an int or np.int64, actual type: \u001b[0m\n", - " logger.warn(f\"{pre} should be an int or np.int64, actual type: {type(obs)}\")\n", - "/home/tk/.virtualenvs/humemai/lib/python3.10/site-packages/gymnasium/utils/passive_env_checker.py:159: UserWarning: \u001b[33mWARN: The obs returned by the `step()` method is not within the observation space.\u001b[0m\n", - " logger.warn(f\"{pre} is not within the observation space.\")\n" - ] - } - ], + "outputs": [], "source": [ "import gymnasium as gym\n", "import random\n", "\n", - "env = gym.make(\"room_env:RoomEnv-v2\", room_size=\"xxl\")\n", + "env = gym.make(\"room_env:RoomEnv-v2\", room_size=\"l\")\n", "observations, info = env.reset()\n", "rewards = 0\n", "\n", @@ -579,7 +560,9 @@ " )\n", " rewards += reward\n", " if done or truncated:\n", - " break" + " break\n", + "\n", + "room_layout = env.unwrapped.return_room_layout(exclude_walls=True)" ] } ], diff --git a/room_env/envs/room2.py b/room_env/envs/room2.py index 1187c00..c816a1e 100644 --- a/room_env/envs/room2.py +++ b/room_env/envs/room2.py @@ -17,7 +17,6 @@ from ..utils import read_json_prod as read_json from ..utils import sample_max_value_key, seed_everything - EPSILON = 1e-3 diff --git a/test/room_env2/test_room_env_v2.py b/test/room_env2/test_room_env_v2.py index d11560d..f25d26f 100644 --- a/test/room_env2/test_room_env_v2.py +++ b/test/room_env2/test_room_env_v2.py @@ -467,3 +467,38 @@ def test_all(self) -> None: self.assertEqual(observations["room"][4][0], "agent") for obs in observations["room"][5:]: self.assertNotEqual(obs[0], "agent") + + +class RoomEnv2LayoutTest(unittest.TestCase): + def setUp(self) -> None: + self.env = gym.make( + "room_env:RoomEnv-v2", + room_size="l", + randomize_observations="none", + include_walls_in_observations=True, + num_total_questions=100, + ) + + def test_room_layout(self) -> None: + observations, info = self.env.reset() + + room_layout = self.env.unwrapped.return_room_layout(exclude_walls=True) + + heads = [triple[0] for triple in room_layout] + sorted_heads = sorted(heads) + self.assertEqual(sorted_heads, heads) + + tails = [triple[2] for triple in room_layout] + num_walls = tails.count("wall") + self.assertEqual(num_walls, 0) + + room_layout = self.env.unwrapped.return_room_layout(exclude_walls=False) + heads = [triple[0] for triple in room_layout] + sorted_heads = sorted(heads) + self.assertEqual(sorted_heads, heads) + + tails_ = [triple[2] for triple in room_layout] + num_walls_ = tails_.count("wall") + self.assertNotEqual(num_walls_, 0) + + self.assertGreater(num_walls_, num_walls)