-
Notifications
You must be signed in to change notification settings - Fork 21
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
19 changed files
with
482 additions
and
171 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,28 +1,51 @@ | ||
| from gym_csle_cyborg.dao.csle_cyborg_config import CSLECyborgConfig | ||
| from gym_csle_cyborg.dao.red_agent_type import RedAgentType | ||
| from gym_csle_cyborg.envs.cyborg_scenario_two_defender import CyborgScenarioTwoDefender | ||
| from gym_csle_cyborg.dao.blue_agent_action_type import BlueAgentActionType | ||
| import gym_csle_cyborg.constants.constants as env_constants | ||
|
|
||
| if __name__ == '__main__': | ||
| config = CSLECyborgConfig( | ||
| gym_env_name="csle-cyborg-scenario-two-v1", scenario=2, baseline_red_agents=[RedAgentType.B_LINE_AGENT], | ||
| maximum_steps=100, red_agent_distribution=[1.0], reduced_action_space=False, decoy_state=False, | ||
| scanned_state=False, decoy_optimization=False) | ||
| maximum_steps=100, red_agent_distribution=[1.0], reduced_action_space=True, decoy_state=True, | ||
| scanned_state=True, decoy_optimization=False) | ||
| csle_cyborg_env = CyborgScenarioTwoDefender(config=config) | ||
| a = 1 | ||
| R = 0 | ||
| for t in range(1000): | ||
| o, r, done, _, info = csle_cyborg_env.step(a) | ||
| if done: | ||
| csle_cyborg_env.reset() | ||
| R += r | ||
| print(f"time-step: {t + 1}, cumulative reward: {R}, a: {a}") | ||
| for i in range(len(info["obs_per_host"])): | ||
| if csle_cyborg_env.cyborg_hostnames[i] == "User0": | ||
| continue | ||
| a = 1 | ||
| if info["obs_per_host"][i]["compromised"].value > 0: | ||
| host = csle_cyborg_env.cyborg_hostnames[i] | ||
| action_type = BlueAgentActionType.RESTORE | ||
| a = csle_cyborg_env.cyborg_action_type_and_host_to_id[(action_type, host)] | ||
| break | ||
| o, info = csle_cyborg_env.reset() | ||
| initial_state_id = info[env_constants.ENV_METRICS.STATE] | ||
| csle_cyborg_env.step(1) | ||
| csle_cyborg_env.set_state(state=initial_state_id) | ||
| # print(csle_cyborg_env.cyborg_challenge_env.env.env.env.env.env.environment_controller.observation["Red"].data["User0"]) | ||
| csle_cyborg_env.step(1) | ||
|
|
||
| # print("INITIAL2 STATE") | ||
| # print(csle_cyborg_env.get_true_table()) | ||
| # # csle_cyborg_env.get_true_table() | ||
| # o, r, done, _, info = csle_cyborg_env.step(1) | ||
| # print("INITIAL1 STATE") | ||
| # print(csle_cyborg_env.get_true_table()) | ||
| # initial_obs_id = info[env_constants.ENV_METRICS.OBSERVATION] | ||
| # initial_state_id = info[env_constants.ENV_METRICS.STATE] | ||
| # # csle_cyborg_env.set_state(state=initial_state_id) | ||
| # csle_cyborg_env.step(1) | ||
| # print("SECOND STATE") | ||
| # print(csle_cyborg_env.get_true_table()) | ||
| # csle_cyborg_env.step(1) | ||
| # csle_cyborg_env.step(1) | ||
| # csle_cyborg_env.step(1) | ||
| # csle_cyborg_env.step(1) | ||
| # csle_cyborg_env.step(1) | ||
| # csle_cyborg_env.step(1) | ||
| # print(csle_cyborg_env.get_true_table()) | ||
| # print("SET STATE") | ||
| # csle_cyborg_env.set_state(state=initial_state_id) | ||
| # print(csle_cyborg_env.get_true_table()) | ||
| # csle_cyborg_env.step(1) | ||
| # print(csle_cyborg_env.get_true_table()) | ||
| # csle_cyborg_env.step(1) | ||
| # print(csle_cyborg_env.get_true_table()) | ||
| # csle_cyborg_env.step(1) | ||
| # print(csle_cyborg_env.get_true_table()) | ||
| # csle_cyborg_env.step(1) | ||
| # csle_cyborg_env.step(1) | ||
| # csle_cyborg_env.step(1) | ||
| # csle_cyborg_env.step(1) | ||
| # print(csle_cyborg_env.get_true_table()) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
20 changes: 20 additions & 0 deletions
20
examples/training/pomcp/cyborg_scenario_two_defender/README.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,20 @@ | ||
| # POMCP for defender plannign in cyborg | ||
|
|
||
| ## Commands | ||
|
|
||
| To run a script, execute: | ||
| ```bash | ||
| python <script_name> | ||
| ``` | ||
|
|
||
| ## Author & Maintainer | ||
|
|
||
| Kim Hammar <kimham@kth.se> | ||
|
|
||
| ## Copyright and license | ||
|
|
||
| [LICENSE](../../../../LICENSE.md) | ||
|
|
||
| Creative Commons | ||
|
|
||
| (C) 2020-2024, Kim Hammar |
89 changes: 89 additions & 0 deletions
89
examples/training/pomcp/cyborg_scenario_two_defender/run_vs_random_attacker_v_001.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,89 @@ | ||
| import csle_common.constants.constants as constants | ||
| from csle_common.dao.training.experiment_config import ExperimentConfig | ||
| from csle_common.metastore.metastore_facade import MetastoreFacade | ||
| from csle_common.dao.training.agent_type import AgentType | ||
| from csle_common.dao.training.hparam import HParam | ||
| from csle_common.dao.training.player_type import PlayerType | ||
| from csle_agents.agents.pomcp.pomcp_agent import POMCPAgent | ||
| import csle_agents.constants.constants as agents_constants | ||
| from csle_agents.common.objective_type import ObjectiveType | ||
| from gym_csle_cyborg.dao.csle_cyborg_config import CSLECyborgConfig | ||
| from gym_csle_cyborg.dao.red_agent_type import RedAgentType | ||
| from gym_csle_cyborg.envs.cyborg_scenario_two_defender import CyborgScenarioTwoDefender | ||
|
|
||
| if __name__ == '__main__': | ||
| emulation_name = "csle-level9-040" | ||
| emulation_env_config = MetastoreFacade.get_emulation_by_name(emulation_name) | ||
| if emulation_env_config is None: | ||
| raise ValueError(f"Could not find an emulation environment with the name: {emulation_name}") | ||
| simulation_name = "csle-cyborg-001" | ||
| simulation_env_config = MetastoreFacade.get_simulation_by_name(simulation_name) | ||
| if simulation_env_config is None: | ||
| raise ValueError(f"Could not find a simulation with name: {simulation_name}") | ||
| simulation_env_config.simulation_env_input_config = CSLECyborgConfig( | ||
| gym_env_name="csle-cyborg-scenario-two-v1", scenario=2, baseline_red_agents=[RedAgentType.B_LINE_AGENT], | ||
| maximum_steps=100, red_agent_distribution=[1.0], reduced_action_space=True, scanned_state=True, | ||
| decoy_state=True, decoy_optimization=False) | ||
| csle_cyborg_env = CyborgScenarioTwoDefender(config=simulation_env_config.simulation_env_input_config) | ||
| A = csle_cyborg_env.get_action_space() | ||
| b1 = csle_cyborg_env.initial_belief | ||
| experiment_config = ExperimentConfig( | ||
| output_dir=f"{constants.LOGGING.DEFAULT_LOG_DIR}pomcp_test", title="POMCP test", | ||
| random_seeds=[399, 98912, 999, 555], | ||
| agent_type=AgentType.POMCP, | ||
| log_every=1, | ||
| hparams={ | ||
| agents_constants.POMCP.N: HParam(value=50, name=agents_constants.POMCP.N, | ||
| descr="the number of episodes"), | ||
| agents_constants.POMCP.OBJECTIVE_TYPE: HParam( | ||
| value=ObjectiveType.MAX, name=agents_constants.POMCP.OBJECTIVE_TYPE, | ||
| descr="the type of objective (max or min)"), | ||
| agents_constants.POMCP.ROLLOUT_POLICY: HParam( | ||
| value=None, name=agents_constants.POMCP.ROLLOUT_POLICY, | ||
| descr="the policy to use for rollouts"), | ||
| agents_constants.POMCP.VALUE_FUNCTION: HParam( | ||
| value=lambda x: 0, name=agents_constants.POMCP.VALUE_FUNCTION, | ||
| descr="the value function to use for truncated rollouts"), | ||
| agents_constants.POMCP.A: HParam(value=A, name=agents_constants.POMCP.A, descr="the action space"), | ||
| agents_constants.POMCP.GAMMA: HParam(value=0.99, name=agents_constants.POMCP.GAMMA, | ||
| descr="the discount factor"), | ||
| agents_constants.POMCP.REINVIGORATION: HParam(value=False, name=agents_constants.POMCP.REINVIGORATION, | ||
| descr="whether reinvigoration should be used"), | ||
| agents_constants.POMCP.INITIAL_BELIEF: HParam(value=b1, name=agents_constants.POMCP.INITIAL_BELIEF, | ||
| descr="the initial belief"), | ||
| agents_constants.POMCP.PLANNING_TIME: HParam(value=300, name=agents_constants.POMCP.PLANNING_TIME, | ||
| descr="the planning time"), | ||
| agents_constants.POMCP.MAX_PARTICLES: HParam(value=1000, name=agents_constants.POMCP.MAX_PARTICLES, | ||
| descr="the maximum number of belief particles"), | ||
| agents_constants.POMCP.MAX_DEPTH: HParam(value=500, name=agents_constants.POMCP.MAX_DEPTH, | ||
| descr="the maximum depth for planning"), | ||
| agents_constants.POMCP.C: HParam(value=0.35, name=agents_constants.POMCP.C, | ||
| descr="the weighting factor for UCB exploration"), | ||
| agents_constants.POMCP.LOG_STEP_FREQUENCY: HParam( | ||
| value=1, name=agents_constants.POMCP.LOG_STEP_FREQUENCY, descr="frequency of logging time-steps"), | ||
| agents_constants.POMCP.DEFAULT_NODE_VALUE: HParam( | ||
| value=-2000, name=agents_constants.POMCP.DEFAULT_NODE_VALUE, descr="the default node value in " | ||
| "the search tree"), | ||
| agents_constants.POMCP.VERBOSE: HParam(value=True, name=agents_constants.POMCP.VERBOSE, | ||
| descr="verbose logging flag"), | ||
| agents_constants.COMMON.EVAL_BATCH_SIZE: HParam(value=100, name=agents_constants.COMMON.EVAL_BATCH_SIZE, | ||
| descr="number of evaluation episodes"), | ||
| agents_constants.COMMON.CONFIDENCE_INTERVAL: HParam( | ||
| value=0.95, name=agents_constants.COMMON.CONFIDENCE_INTERVAL, | ||
| descr="confidence interval"), | ||
| agents_constants.COMMON.MAX_ENV_STEPS: HParam( | ||
| value=500, name=agents_constants.COMMON.MAX_ENV_STEPS, | ||
| descr="maximum number of steps in the environment (for envs with infinite horizon generally)"), | ||
| agents_constants.COMMON.RUNNING_AVERAGE: HParam( | ||
| value=100, name=agents_constants.COMMON.RUNNING_AVERAGE, | ||
| descr="the number of samples to include when computing the running avg"), | ||
| agents_constants.COMMON.GAMMA: HParam( | ||
| value=0.99, name=agents_constants.COMMON.GAMMA, | ||
| descr="the discount factor") | ||
| }, | ||
| player_type=PlayerType.DEFENDER, player_idx=0 | ||
| ) | ||
| agent = POMCPAgent(emulation_env_config=emulation_env_config, simulation_env_config=simulation_env_config, | ||
| experiment_config=experiment_config, save_to_metastore=False) | ||
| experiment_execution = agent.train() | ||
| MetastoreFacade.save_experiment_execution(experiment_execution) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.