Feature/atari wrapper (#92)

* implement wrapper * update config and default value * fix minor * Update conftest.py * update wrapper code * 🎨 Format Python code with psf/black (#95) Co-authored-by: ramanuzan <ramanuzan@users.noreply.github.com> * update skip frame buffer * update log file name %Y%m%d%H%M%S -> %Y%m%d%H%M%S%f * update denominator * update config * update fire reset assertion * apply black Co-authored-by: root <root@gpu-cloud-vnode53.dakao.io> Co-authored-by: root <root@brain-cluster-gpu9.dakao.io> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: ramanuzan <ramanuzan@users.noreply.github.com> Co-authored-by: root <root@gpu-cloud-node5.dakao.io>
kakaoenterprise · Jan 3, 2022 · e70f9e3 · e70f9e3
1 parent 24f1c5a
commit e70f9e3
Show file tree

Hide file tree

Showing 79 changed files with 274 additions and 172 deletions.
diff --git a/jorldy/config/ape_x/atari.py b/jorldy/config/ape_x/atari.py
@@ -7,9 +7,10 @@
     "img_width": 84,
     "img_height": 84,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
-    "dead_penalty": True,
+    "episodic_life": True,
 }
 
 agent = {

diff --git a/jorldy/config/ape_x/pong_mlagent.py b/jorldy/config/ape_x/pong_mlagent.py
@@ -1,6 +1,6 @@
 ### Ape-X Pong_ML-Agents Config ###
 
-env = {"name": "pong_mlagent", "train_mode": True}
+env = {"name": "pong_mlagent", "time_scale": 12.0}
 
 agent = {
     "name": "ape_x",

diff --git a/jorldy/config/ape_x/procgen.py b/jorldy/config/ape_x/procgen.py
@@ -5,7 +5,8 @@
     "render": False,
     "gray_img": True,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
 }
 

diff --git a/jorldy/config/c51/atari.py b/jorldy/config/c51/atari.py
@@ -7,9 +7,10 @@
     "img_width": 84,
     "img_height": 84,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
-    "dead_penalty": True,
+    "episodic_life": True,
 }
 
 agent = {

diff --git a/jorldy/config/c51/pong_mlagent.py b/jorldy/config/c51/pong_mlagent.py
@@ -1,6 +1,6 @@
 ### C51 Pong_ML-Agents Config ###
 
-env = {"name": "pong_mlagent", "train_mode": True}
+env = {"name": "pong_mlagent", "time_scale": 12.0}
 
 agent = {
     "name": "c51",

diff --git a/jorldy/config/c51/procgen.py b/jorldy/config/c51/procgen.py
@@ -5,7 +5,8 @@
     "render": False,
     "gray_img": True,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
 }
 

diff --git a/jorldy/config/ddpg/hopper_mlagent.py b/jorldy/config/ddpg/hopper_mlagent.py
@@ -1,6 +1,6 @@
 ### DDPG Hopper Config ###
 
-env = {"name": "hopper_mlagent", "train_mode": True}
+env = {"name": "hopper_mlagent", "time_scale": 12.0}
 
 agent = {
     "name": "ddpg",

diff --git a/jorldy/config/double/atari.py b/jorldy/config/double/atari.py
@@ -7,9 +7,10 @@
     "img_width": 84,
     "img_height": 84,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
-    "dead_penalty": True,
+    "episodic_life": True,
 }
 
 agent = {

diff --git a/jorldy/config/double/pong_mlagent.py b/jorldy/config/double/pong_mlagent.py
@@ -1,6 +1,6 @@
 ### Double DQN Pong MLAgent Config ###
 
-env = {"name": "pong_mlagent", "train_mode": True}
+env = {"name": "pong_mlagent", "time_scale": 12.0}
 
 agent = {
     "name": "double",

diff --git a/jorldy/config/double/procgen.py b/jorldy/config/double/procgen.py
@@ -5,7 +5,8 @@
     "render": False,
     "gray_img": True,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
 }
 

diff --git a/jorldy/config/dqn/atari.py b/jorldy/config/dqn/atari.py
@@ -7,9 +7,10 @@
     "img_width": 84,
     "img_height": 84,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
-    "dead_penalty": True,
+    "episodic_life": True,
 }
 
 agent = {
@@ -27,19 +28,19 @@
 }
 
 optim = {
-    "name": "rmsprop",
-    "lr": 2.5e-4,
+    "name": "adam",
+    "lr": 1e-4,
 }
 
 train = {
     "training": True,
     "load_path": None,
-    "run_step": 30000000,
+    "run_step": 10000000,
     "print_period": 10000,
     "save_period": 100000,
     "eval_iteration": 5,
     "record": True,
-    "record_period": 300000,
+    "record_period": 500000,
     # distributed setting
     "update_period": 32,
     "num_workers": 16,

diff --git a/jorldy/config/dqn/pong_mlagent.py b/jorldy/config/dqn/pong_mlagent.py
@@ -1,6 +1,6 @@
 ### DQN Pong_ML-Agents Config ###
 
-env = {"name": "pong_mlagent", "train_mode": True}
+env = {"name": "pong_mlagent", "time_scale": 12.0}
 
 agent = {
     "name": "dqn",

diff --git a/jorldy/config/dqn/procgen.py b/jorldy/config/dqn/procgen.py
@@ -5,7 +5,8 @@
     "render": False,
     "gray_img": True,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
 }
 

diff --git a/jorldy/config/dueling/atari.py b/jorldy/config/dueling/atari.py
@@ -7,9 +7,10 @@
     "img_width": 84,
     "img_height": 84,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
-    "dead_penalty": True,
+    "episodic_life": True,
 }
 
 agent = {

diff --git a/jorldy/config/dueling/pong_mlagent.py b/jorldy/config/dueling/pong_mlagent.py
@@ -1,6 +1,6 @@
 ### Dueling DQN Pong_ML-Agents Config ###
 
-env = {"name": "pong_mlagent", "train_mode": True}
+env = {"name": "pong_mlagent", "time_scale": 12.0}
 
 agent = {
     "name": "dueling",

diff --git a/jorldy/config/dueling/procgen.py b/jorldy/config/dueling/procgen.py
@@ -5,7 +5,8 @@
     "render": False,
     "gray_img": True,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
 }
 

diff --git a/jorldy/config/icm_ppo/atari.py b/jorldy/config/icm_ppo/atari.py
@@ -7,9 +7,10 @@
     "img_width": 84,
     "img_height": 84,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
-    "dead_penalty": True,
+    "episodic_life": True,
 }
 
 agent = {

diff --git a/jorldy/config/icm_ppo/drone_delivery_mlagent.py b/jorldy/config/icm_ppo/drone_delivery_mlagent.py
@@ -1,6 +1,6 @@
 ### ICM PPO Drone Delivery Config ###
 
-env = {"name": "drone_delivery_mlagent", "train_mode": True}
+env = {"name": "drone_delivery_mlagent", "time_scale": 12.0}
 
 agent = {
     "name": "icm_ppo",

diff --git a/jorldy/config/icm_ppo/pong_mlagent.py b/jorldy/config/icm_ppo/pong_mlagent.py
@@ -1,6 +1,6 @@
 ### ICM PPO Pong_ML-Agents Config ###
 
-env = {"name": "pong_mlagent", "train_mode": True}
+env = {"name": "pong_mlagent", "time_scale": 12.0}
 
 agent = {
     "name": "icm_ppo",

diff --git a/jorldy/config/icm_ppo/procgen.py b/jorldy/config/icm_ppo/procgen.py
@@ -5,7 +5,8 @@
     "render": False,
     "gray_img": True,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
 }
 

diff --git a/jorldy/config/icm_ppo/super_mario_bros.py b/jorldy/config/icm_ppo/super_mario_bros.py
@@ -7,9 +7,10 @@
     "img_width": 84,
     "img_height": 84,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
-    "dead_penalty": True,
+    "episodic_life": True,
 }
 
 agent = {

diff --git a/jorldy/config/iqn/atari.py b/jorldy/config/iqn/atari.py
@@ -7,9 +7,10 @@
     "img_width": 84,
     "img_height": 84,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
-    "dead_penalty": True,
+    "episodic_life": True,
 }
 
 agent = {

diff --git a/jorldy/config/iqn/pong_mlagent.py b/jorldy/config/iqn/pong_mlagent.py
@@ -1,6 +1,6 @@
 ### IQN Pong_ML-Agents Config ###
 
-env = {"name": "pong_mlagent", "train_mode": True}
+env = {"name": "pong_mlagent", "time_scale": 12.0}
 
 agent = {
     "name": "iqn",

diff --git a/jorldy/config/iqn/procgen.py b/jorldy/config/iqn/procgen.py
@@ -5,7 +5,8 @@
     "render": False,
     "gray_img": True,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
 }
 

diff --git a/jorldy/config/m_dqn/atari.py b/jorldy/config/m_dqn/atari.py
@@ -8,8 +8,9 @@
     "img_height": 84,
     "stack_frame": 4,
     "no_op": True,
-    "reward_clip": False,
-    "dead_penalty": False,
+    "skip_frame": 4,
+    "reward_clip": True,
+    "episodic_life": True,
 }
 
 agent = {

diff --git a/jorldy/config/m_dqn/pong_mlagent.py b/jorldy/config/m_dqn/pong_mlagent.py
@@ -1,6 +1,6 @@
 ### Munchausen DQN Pong_ML-Agents Config ###
 
-env = {"name": "pong_mlagent", "train_mode": True}
+env = {"name": "pong_mlagent", "time_scale": 12.0}
 
 agent = {
     "name": "m_dqn",

diff --git a/jorldy/config/m_dqn/procgen.py b/jorldy/config/m_dqn/procgen.py
@@ -5,7 +5,8 @@
     "render": False,
     "gray_img": True,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
 }
 

diff --git a/jorldy/config/m_iqn/atari.py b/jorldy/config/m_iqn/atari.py
@@ -7,9 +7,10 @@
     "img_width": 84,
     "img_height": 84,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
-    "dead_penalty": True,
+    "episodic_life": True,
 }
 
 agent = {

diff --git a/jorldy/config/m_iqn/pong_mlagent.py b/jorldy/config/m_iqn/pong_mlagent.py
@@ -1,6 +1,6 @@
 ### Munchausen IQN Pong_ML-Agents Config ###
 
-env = {"name": "pong_mlagent", "train_mode": True}
+env = {"name": "pong_mlagent", "time_scale": 12.0}
 
 agent = {
     "name": "m_iqn",

diff --git a/jorldy/config/m_iqn/procgen.py b/jorldy/config/m_iqn/procgen.py
@@ -5,7 +5,8 @@
     "render": False,
     "gray_img": True,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
 }
 

diff --git a/jorldy/config/mpo/atari.py b/jorldy/config/mpo/atari.py
@@ -7,9 +7,10 @@
     "img_width": 84,
     "img_height": 84,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
-    "dead_penalty": True,
+    "episodic_life": True,
 }
 
 agent = {

diff --git a/jorldy/config/mpo/drone_delivery_mlagent.py b/jorldy/config/mpo/drone_delivery_mlagent.py
@@ -1,6 +1,6 @@
 ### MPO Drone Delivery Config ###
 
-env = {"name": "drone_delivery_mlagent", "train_mode": True}
+env = {"name": "drone_delivery_mlagent", "time_scale": 12.0}
 
 agent = {
     "name": "mpo",

diff --git a/jorldy/config/mpo/hopper_mlagent.py b/jorldy/config/mpo/hopper_mlagent.py
@@ -1,6 +1,6 @@
 ### MPO Hopper Config ###
 
-env = {"name": "hopper_mlagent", "train_mode": True}
+env = {"name": "hopper_mlagent", "time_scale": 12.0}
 
 agent = {
     "name": "mpo",

diff --git a/jorldy/config/mpo/pong_mlagent.py b/jorldy/config/mpo/pong_mlagent.py
@@ -1,6 +1,6 @@
 ### MPO Pong_ML-Agents Config ###
 
-env = {"name": "pong_mlagent", "train_mode": True}
+env = {"name": "pong_mlagent", "time_scale": 12.0}
 
 agent = {
     "name": "mpo",

diff --git a/jorldy/config/mpo/procgen.py b/jorldy/config/mpo/procgen.py
@@ -5,7 +5,8 @@
     "render": False,
     "gray_img": True,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
 }
 

diff --git a/jorldy/config/multistep/atari.py b/jorldy/config/multistep/atari.py
@@ -7,9 +7,10 @@
     "img_width": 84,
     "img_height": 84,
     "stack_frame": 4,
-    "no_op": False,
+    "no_op": True,
+    "skip_frame": 4,
     "reward_clip": True,
-    "dead_penalty": True,
+    "episodic_life": True,
 }
 
 agent = {

diff --git a/jorldy/config/multistep/pong_mlagent.py b/jorldy/config/multistep/pong_mlagent.py
@@ -1,6 +1,6 @@
 ### Multistep DQN Pong_ML-Agents Config ###
 
-env = {"name": "pong_mlagent", "train_mode": True}
+env = {"name": "pong_mlagent", "time_scale": 12.0}
 
 agent = {
     "name": "multistep",