diff --git a/examples/plotting/orlando_plotting.py b/examples/plotting/orlando_plotting.py index d1a72ff8c..dc016a0d6 100644 --- a/examples/plotting/orlando_plotting.py +++ b/examples/plotting/orlando_plotting.py @@ -19,10 +19,6 @@ def plot_returns(returns_means, returns_stds, file_name: str, fontsize: int = 18 plt.rcParams.update({'font.size': fontsize}) plt.rcParams['font.family'] = ['serif'] fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(11, 4)) - # ax.plot(np.array(list(range(len(returns_means))))*10, returns_means, label=r"$\pi^{(i)}_D$ simulation", - # marker="o", ls='-', color="black", markevery=1, markersize=3, lw=0.75) - # ax.fill_between(np.array(list(range(len(returns_means))))*10, returns_means - returns_stds, - # returns_means + returns_stds, alpha=0.35, color="black", lw=0.75) ax.plot(np.array(list(range(len(returns_means)))) * 200, returns_means, label=r"$\pi^{(i)}_D$ simulation", marker="o", ls='-', color="black", markevery=1, markersize=3, lw=0.75) @@ -32,10 +28,6 @@ def plot_returns(returns_means, returns_stds, file_name: str, fontsize: int = 18 ax.spines['right'].set_visible(False) ax.set_xlabel(r"iteration $k$") ax.set_ylabel(r"Avg cumulative reward") - # ax.legend(loc='upper center', bbox_to_anchor=(0.51, -0.15), - # ncol=4, fancybox=True, shadow=False, handletextpad=0.4, labelspacing=0.5, columnspacing=0.65, - # fontsize=fontsize) - # ax.set_title(r"\textsc{ppo-base}-1") ax.set_title(r"\textsc{dqn-base}-1") fig.tight_layout() fig.subplots_adjust(wspace=0.0, hspace=0.75) @@ -58,11 +50,8 @@ def plot_returns(returns_means, returns_stds, file_name: str, fontsize: int = 18 metrics = experiment_result.all_metrics[seed] returns = metrics["average_return"] folder = "/home/kim/orlando_results/12_jan/" - # plot_file_name = "ppo_base_1" result_file_name = "ppo_base_1.json" plot_file_name = "dqn_base_1" experiment_result.to_json_file(f"{folder}/{result_file_name}") - # plot_returns(returns_means=np.array(returns), returns_stds=np.array([0]*len(returns)), - # file_name=f"{folder}{plot_file_name}") plot_returns(returns_means=np.array(dqn_results), returns_stds=np.array([0] * len(dqn_results)), file_name=f"{folder}{plot_file_name}") diff --git a/examples/plotting/plot_learning_curves_tolerance.py b/examples/plotting/plot_learning_curves_tolerance.py index c8fc490f7..94e6157b1 100644 --- a/examples/plotting/plot_learning_curves_tolerance.py +++ b/examples/plotting/plot_learning_curves_tolerance.py @@ -3,12 +3,6 @@ from csle_common.metastore.metastore_facade import MetastoreFacade if __name__ == '__main__': - # cross_entropy_ids = MetastoreFacade.list_experiment_executions_ids() - # for id in cross_entropy_ids: - # ex = MetastoreFacade.get_experiment_execution(id=id[0]) - # print(f"id: {ex.id}, agent: {ex.config.agent_type}, BTR: {ex.config.hparams['L'].value}") - # import sys - # sys.exit(0) fontsize: int = 14 lw: float = 0.75 alpha: float = 0.35 @@ -54,32 +48,8 @@ col += 1 if col >= 8: col = 0 - # for i in range(len(cross_entropy_ids)): - # BTR = 11 + i - # ax[1][i].spines['top'].set_visible(False) - # ax[1][i].spines['right'].set_visible(False) - # ax[1][i].set_xlabel(r"Time (min)", fontsize=fontsize) - # if i == 0: - # ax[1][i].set_ylabel(r"Average cost $J_i$", fontsize=fontsize) - # ax[1][i].set_title(r"$\Delta_{\mathrm{R}}=" + str(BTR) + "$", fontsize=fontsize) - # ax[1][i].tick_params(axis='both', which='major', labelsize=fontsize) - # ax[1][i].tick_params(axis='both', which='minor', labelsize=fontsize) - # ax[1][i].set_xlabel(r"Time (min)", fontsize=fontsize) - # for i in range(len(cross_entropy_ids)): - # BTR = 11 + i - # ax[2][i].spines['top'].set_visible(False) - # ax[2][i].spines['right'].set_visible(False) - # ax[2][i].set_xlabel(r"Time (min)", fontsize=fontsize) - # if i == 0: - # ax[2][i].set_ylabel(r"Average cost $J_i$", fontsize=fontsize) - # ax[2][i].set_title(r"$\Delta_{\mathrm{R}}=" + str(BTR) + "$", fontsize=fontsize) - # ax[2][i].tick_params(axis='both', which='major', labelsize=fontsize) - # ax[2][i].tick_params(axis='both', which='minor', labelsize=fontsize) - # ax[2][i].set_xlabel(r"Time (min)", fontsize=fontsize) fig.tight_layout() fig.subplots_adjust(wspace=0.55, hspace=0.4, bottom=0.14) - # fig.savefig(file_name + ".png", format="png", dpi=600) - # fig.savefig(file_name + ".pdf", format='pdf', dpi=600, bbox_inches='tight', transparent=True) handles, labels = ax[0][0].get_legend_handles_labels() fig.legend(handles, labels, loc='lower center', bbox_to_anchor=(0.52, -0.02), ncol=8, fancybox=False, shadow=False, handletextpad=0.4, labelspacing=0.5, columnspacing=0.65, diff --git a/examples/training/pomdp_solve/README.md b/examples/training/pomdp_solve/README.md new file mode 100644 index 000000000..b71c55147 --- /dev/null +++ b/examples/training/pomdp_solve/README.md @@ -0,0 +1,25 @@ +# pomdp-solve + +This directory contains example scripts for solving pomdps using [pomdp-solve](https://www.pomdp.org/code/index.html). + +Command for running pomdp-solve with infinite time horizon with a discount factor: +```bash +nohup pomdp-solve -pomdp intrusion_recovery.pomdp -discount 0.995 -method incprune > inf_gamma_099.log & +``` + +Command for running pomdp-solve with a fixed time horizon and no discount: +```bash +nohup pomdp-solve -pomdp intrusion_recovery.pomdp -horizon 100 -method incprune > 100_solve.log & +``` + +## Author & Maintainer + +Kim Hammar + +## Copyright and license + +[LICENSE](../../../LICENSE.md) + +Creative Commons + +(C) 2020-2024, Kim Hammar \ No newline at end of file diff --git a/examples/training/pomdp_solve/intrusion_recovery_pomdp/run_vs_random_attacker_v_001.py b/examples/training/pomdp_solve/intrusion_recovery_pomdp/run_vs_random_attacker_v_001.py new file mode 100644 index 000000000..951c67361 --- /dev/null +++ b/examples/training/pomdp_solve/intrusion_recovery_pomdp/run_vs_random_attacker_v_001.py @@ -0,0 +1,58 @@ +import numpy as np +from csle_tolerance.dao.intrusion_recovery_pomdp_config import IntrusionRecoveryPomdpConfig +from csle_tolerance.util.intrusion_recovery_pomdp_util import IntrusionRecoveryPomdpUtil +from csle_tolerance.util.pomdp_solve_parser import PomdpSolveParser + + +if __name__ == '__main__': + eta = 2 + p_a = 0.05 + p_c_1 = 0.01 + p_c_2 = 0.01 + p_u = 0.0 + BTR = np.inf + negate_costs = False + discount_factor = 1-p_c_1 + num_observations = 100 + simulation_name = "csle-tolerance-intrusion-recovery-pomdp-defender-001" + cost_tensor = IntrusionRecoveryPomdpUtil.cost_tensor(eta=eta, states=IntrusionRecoveryPomdpUtil.state_space(), + actions=IntrusionRecoveryPomdpUtil.action_space(), + negate=negate_costs) + observation_tensor = IntrusionRecoveryPomdpUtil.observation_tensor( + states=IntrusionRecoveryPomdpUtil.state_space(), + observations=IntrusionRecoveryPomdpUtil.observation_space(num_observations=num_observations)) + transition_tensor = IntrusionRecoveryPomdpUtil.transition_tensor( + states=IntrusionRecoveryPomdpUtil.state_space(), actions=IntrusionRecoveryPomdpUtil.action_space(), p_a=p_a, + p_c_1=p_c_1, p_c_2=p_c_2, p_u=p_u) + config = IntrusionRecoveryPomdpConfig( + eta=eta, p_a=p_a, p_c_1=p_c_1, p_c_2=p_c_2, p_u=p_u, BTR=BTR, negate_costs=negate_costs, seed=999, + discount_factor=discount_factor, states=IntrusionRecoveryPomdpUtil.state_space(), + actions=IntrusionRecoveryPomdpUtil.action_space(), + observations=IntrusionRecoveryPomdpUtil.observation_space(num_observations=num_observations), + cost_tensor=cost_tensor, observation_tensor=observation_tensor, transition_tensor=transition_tensor, + b1=IntrusionRecoveryPomdpUtil.initial_belief(p_a=p_a), T=BTR, + simulation_env_name=simulation_name, gym_env_name="csle-tolerance-intrusion-recovery-pomdp-v1" + ) + pomdp_solve_file_str = IntrusionRecoveryPomdpUtil.pomdp_solver_file(config=config) + with open("/home/kim/gamesec24/intrusion_recovery.pomdp", 'w') as f: + f.write(pomdp_solve_file_str) + + alpha_vectors = PomdpSolveParser.parse_alpha_vectors( + file_path="/home/kim/gamesec24/intrusion_recovery-3361312.alpha") + belief_space = np.linspace(0.0, 1, int(1.0/0.01)) + print(belief_space) + for i in range(len(alpha_vectors)): + print(f"a*:{alpha_vectors[i][0]}, vector: {list(-np.array(alpha_vectors[i][1][0:2]))}") + values_01 = [] + for j, b in enumerate(belief_space): + b_vec = [1-b, b] + dot_vals = [] + for i in range(len(alpha_vectors)): + dot_vals.append(np.dot(b_vec, list(-np.array(alpha_vectors[i][1][0:2])))) + min_index = np.argmin(dot_vals) + values_01.append(dot_vals[min_index]) + vec_dots = [] + print(f"{b} {values_01[-1]}") + for b in belief_space: + b_vec = [1-b, b] + vec_dots.append(-np.dot(b_vec, list(-np.array(alpha_vectors[min_index][1][0:2]))))