Feature/115 sac

jorldy/config/sac/atari.py

@@ -0,0 +1,56 @@
+### SAC Atari Config ###
+
+env = {
+    # "name": it should be defined in the command. ex) python main.py --config config.AGENT.atari --env.name breakout
+    "render": False,
+    "gray_img": True,
+    "img_width": 84,
+    "img_height": 84,
+    "stack_frame": 4,
+    "no_op": True,
+    "skip_frame": 4,
+    "reward_clip": True,
+    "episodic_life": True,
+}
+
+agent = {
+    "name": "sac",
+    "actor": "discrete_policy",
+    "critic": "discrete_q_network",
+    "head": "cnn",
+    "use_dynamic_alpha": True,
+    "gamma": 0.99,
+    "tau": 5e-3,
+    "buffer_size": 1000000,
+    "batch_size": 32,
+    "start_train_step": 100000,
+    "static_log_alpha": -6.0,
+    "target_update_period": 10000,
+}
+
+
+optim = {
+    "actor": "adam",
+    "critic": "adam",
+    "alpha": "adam",
+    # "actor_lr": 5e-4,
+    # "critic_lr": 1e-3,
+    # "alpha_lr": 3e-4,
+    "actor_lr": 1.5e-4,
+    "critic_lr": 3e-4,
+    "alpha_lr": 1e-5,
+}
+
+train = {
+    "training": True,
+    "load_path": None,
+    "run_step": 10000000,
+    "print_period": 10000,
+    "save_period": 100000,
+    "eval_iteration": 5,
+    "record": True,
+    "record_period": 500000,
+    # distributed setting
+    "update_period": 32,
+    "num_workers": 16,
+}

jorldy/config/sac/cartpole.py

@@ -9,23 +9,24 @@
 agent = {
     "name": "sac",
     "actor": "continuous_policy",
-    "critic": "sac_critic",
+    "critic": "continuous_q_network",
     "use_dynamic_alpha": True,
     "gamma": 0.99,
     "tau": 5e-3,
     "buffer_size": 50000,
     "batch_size": 64,
     "start_train_step": 5000,
     "static_log_alpha": -2.0,
+    "target_update_period": 500,
 }
 
 optim = {
     "actor": "adam",
     "critic": "adam",
     "alpha": "adam",
-    "actor_lr": 5e-4,
-    "critic_lr": 1e-3,
-    "alpha_lr": 3e-4,
+    "actor_lr": 1.5e-4,
+    "critic_lr": 3e-4,
+    "alpha_lr": 1e-5,
 }
 
 train = {

jorldy/config/sac/cartpole_discrete.py

@@ -0,0 +1,42 @@
+### SAC CartPole Config ###
+
+env = {
+    "name": "cartpole",
+    "action_type": "discrete",
+    "render": False,
+}
+
+agent = {
+    "name": "sac",
+    "actor": "discrete_policy",
+    "critic": "discrete_q_network",
+    "use_dynamic_alpha": True,
+    "gamma": 0.99,
+    "tau": 5e-3,
+    "buffer_size": 50000,
+    "batch_size": 64,
+    "start_train_step": 5000,
+    "static_log_alpha": -2.0,
+    "target_update_period": 500,
+}
+
+optim = {
+    "actor": "adam",
+    "critic": "adam",
+    "alpha": "adam",
+    "actor_lr": 1.5e-4,
+    "critic_lr": 3e-4,
+    "alpha_lr": 1e-5,
+}
+
+train = {
+    "training": True,
+    "load_path": None,
+    "run_step": 100000,
+    "print_period": 1000,
+    "save_period": 10000,
+    "eval_iteration": 10,
+    # distributed setting
+    "update_period": 32,
+    "num_workers": 8,
+}

jorldy/config/sac/drone_delivery_mlagent.py

@@ -5,7 +5,7 @@
 agent = {
     "name": "sac",
     "actor": "continuous_policy",
-    "critic": "sac_critic",
+    "critic": "continuous_q_network",
     "head": "multi",
     "use_dynamic_alpha": True,
     "gamma": 0.99,

jorldy/config/sac/hopper_mlagent.py

@@ -5,7 +5,7 @@
 agent = {
     "name": "sac",
     "actor": "continuous_policy",
-    "critic": "sac_critic",
+    "critic": "continuous_q_network",
     "use_dynamic_alpha": True,
     "gamma": 0.99,
     "tau": 5e-3,

jorldy/config/sac/mujoco.py

@@ -8,8 +8,7 @@
 agent = {
     "name": "sac",
     "actor": "continuous_policy",
-    "critic": "sac_critic",
-    "head": "multi",
+    "critic": "continuous_q_network",
     "use_dynamic_alpha": True,
     "gamma": 0.99,
     "tau": 5e-3,

jorldy/config/sac/pendulum.py

@@ -8,7 +8,7 @@
 agent = {
     "name": "sac",
     "actor": "continuous_policy",
-    "critic": "sac_critic",
+    "critic": "continuous_q_network",
     "use_dynamic_alpha": True,
     "gamma": 0.99,
     "tau": 5e-3,

jorldy/config/sac/pong_mlagent.py

@@ -0,0 +1,38 @@
+### SAC Pong_ML-Agents Config ###
+
+env = {"name": "pong_mlagent", "time_scale": 12.0}
+
+agent = {
+    "name": "sac",
+    "actor": "discrete_policy",
+    "critic": "discrete_q_network",
+    "use_dynamic_alpha": True,
+    "gamma": 0.99,
+    "tau": 5e-3,
+    "buffer_size": 50000,
+    "batch_size": 32,
+    "start_train_step": 25000,
+    "static_log_alpha": -3.0,
+    "target_update_period": 1000,
+}
+
+optim = {
+    "actor": "adam",
+    "critic": "adam",
+    "alpha": "adam",
+    "actor_lr": 1.5e-4,
+    "critic_lr": 3e-4,
+    "alpha_lr": 1e-5,
+}
+
+train = {
+    "training": True,
+    "load_path": None,
+    "run_step": 200000,
+    "print_period": 5000,
+    "save_period": 50000,
+    "eval_iteration": 10,
+    # distributed setting
+    "update_period": 8,
+    "num_workers": 16,
+}

jorldy/config/sac/procgen.py

@@ -0,0 +1,49 @@
+### SAC Procgen Config ###
+
+env = {
+    # "name": it should be defined in the command. ex) python main.py --config config.dqn.procgen --env.name coinrun
+    "render": False,
+    "gray_img": True,
+    "stack_frame": 4,
+    "no_op": True,
+    "skip_frame": 4,
+    "reward_clip": True,
+}
+
+agent = {
+    "name": "sac",
+    "actor": "discrete_policy",
+    "critic": "discrete_q_network",
+    "head": "cnn",
+    "use_dynamic_alpha": True,
+    "gamma": 0.99,
+    "tau": 5e-3,
+    "buffer_size": 1000000,
+    "batch_size": 32,
+    "start_train_step": 100000,
+    "static_log_alpha": -6.0,
+    "target_update_period": 10000,
+}
+
+optim = {
+    "actor": "adam",
+    "critic": "adam",
+    "alpha": "adam",
+    "actor_lr": 1.5e-4,
+    "critic_lr": 3e-4,
+    "alpha_lr": 1e-5,
+}
+
+train = {
+    "training": True,
+    "load_path": None,
+    "run_step": 30000000,
+    "print_period": 10000,
+    "save_period": 100000,
+    "eval_iteration": 5,
+    "record": True,
+    "record_period": 300000,
+    # distributed setting
+    "update_period": 32,
+    "num_workers": 16,
+}

jorldy/config/sac/super_mario_bros.py

@@ -0,0 +1,47 @@
+### SAC Super Mario Bros Config ###
+
+env = {
+    "name": "super_mario_bros",
+    "render": False,
+    "gray_img": True,
+    "img_width": 84,
+    "img_height": 84,
+    "stack_frame": 4,
+}
+
+agent = {
+    "name": "sac",
+    "actor": "discrete_policy",
+    "critic": "discrete_q_network",
+    "use_dynamic_alpha": True,
+    "gamma": 0.99,
+    "tau": 5e-3,
+    "buffer_size": 50000,
+    "batch_size": 32,
+    "start_train_step": 25000,
+    "static_log_alpha": -3.0,
+    "target_update_period": 1000,
+}
+
+optim = {
+    "actor": "adam",
+    "critic": "adam",
+    "alpha": "adam",
+    "actor_lr": 1.5e-4,
+    "critic_lr": 3e-4,
+    "alpha_lr": 1e-5,
+}
+
+train = {
+    "training": True,
+    "load_path": None,
+    "run_step": 100000000,
+    "print_period": 5000,
+    "save_period": 50000,
+    "eval_iteration": 1,
+    "record": True,
+    "record_period": 200000,
+    # distributed setting
+    "update_period": 32,
+    "num_workers": 16,
+}