diff --git a/Protos/build/Protos.jar b/Protos/build/Protos.jar index 3794ef20..7555a950 100644 Binary files a/Protos/build/Protos.jar and b/Protos/build/Protos.jar differ diff --git a/Protos/src/thinclab/executables/CyberDeceptionAgent.java b/Protos/src/thinclab/executables/CyberDeceptionAgent.java new file mode 100644 index 00000000..1571e832 --- /dev/null +++ b/Protos/src/thinclab/executables/CyberDeceptionAgent.java @@ -0,0 +1,224 @@ +/* + * THINC Lab at UGA | Cyber Deception Group + * + * Author: Aditya Shinde + * + * email: shinde.aditya386@gmail.com + */ +package thinclab.executables; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.CommandLineParser; +import org.apache.commons.cli.DefaultParser; +import org.apache.commons.cli.Options; +import org.apache.log4j.Logger; + +import thinclab.belief.FullBeliefExpansion; +import thinclab.decisionprocesses.IPOMDP; +import thinclab.decisionprocesses.POMDP; +import thinclab.parsers.IPOMDPParser; +import thinclab.representations.conditionalplans.ConditionalPlanTree; +import thinclab.representations.policyrepresentations.PolicyGraph; +import thinclab.simulations.CyberDeceptionSimulation; +import thinclab.solvers.BaseSolver; +import thinclab.solvers.OfflinePBVISolver; +import thinclab.solvers.OfflineSymbolicPerseus; +import thinclab.solvers.OnlineInteractiveSymbolicPerseus; +import thinclab.utils.CustomConfigurationFactory; + +/* + * @author adityas + * + */ +public class CyberDeceptionAgent extends Executable { + + /* + * Defender agent for the cyber deception project + */ + + private static Logger LOGGER; + + // ------------------------------------------------------------------------------------ + + public static void main(String[] args) { + + /* Parse CMD args */ + CommandLineParser cliParser = new DefaultParser(); + Options opt = new Options(); + + /* domain file */ + opt.addOption("d", true, "path to the domain file"); + + /* log file */ + opt.addOption("l", false, "log to file in results dir?"); + + /* backup iterations */ + opt.addOption("b", true, "number of backups in each round"); + + /* solver rounds */ + opt.addOption("r", true, "number of symbolic perseus rounds (always 1 for IPOMDPs)"); + + /* look ahead or search depth */ + opt.addOption("n", true, "look ahead for IPOMDPs / SSGA depth for POMDPs"); + + /* IP address and port of env */ + opt.addOption("a", "ip", true, "IP address for the env server"); + opt.addOption("t", "port", true, "port for the env server"); + + /* simulation switch */ + opt.addOption( + "x", + "interactions", + true, + "run these many iterations"); + + /* results storage directory */ + opt.addOption( + "s", + "output", + true, + "directory where result files are to be stored. (Should be an existing dir)"); + + /* POMDP or IPOMDP switch */ + opt.addOption( + "p", + "pomdp", + false, + "set if the domain is a POMDP domain"); + + opt.addOption( + "i", + "ipomdp", + false, + "set if the domain is a IPOMDP domain"); + + CommandLine line = null; + + try { + line = cliParser.parse(opt, args); + + /* set CLI args */ + + /* output directory */ + String storageDir = line.getOptionValue("s"); + + /* if log file is given, initialize logging accordingly */ + if (line.hasOption("l")) + CustomConfigurationFactory.setLogFileName(storageDir + "/solver.log"); + + CustomConfigurationFactory.initializeLogging(); + LOGGER = Logger.getLogger(RunSimulations.class); + + /* set domain file */ + String domainFile = line.getOptionValue("d"); + + /* set look ahead */ + int backups = new Integer(line.getOptionValue("b")); + + /* set look ahead */ + int lookAhead = new Integer(line.getOptionValue("n")); + + /* get IP address and port of the env server */ + String envIP = line.getOptionValue("a"); + int envPort = new Integer(line.getOptionValue("t")); + + /* conditional plan and policy graph */ + if (line.hasOption("p")) { + LOGGER.info("Simulating POMDP..."); + + int rounds = new Integer(line.getOptionValue("r")); + int simLength = new Integer(line.getOptionValue("x")); + + LOGGER.info("Starting simulation"); + + POMDP pomdp = new POMDP(domainFile); + OfflineSymbolicPerseus solver = + OfflineSymbolicPerseus.createSolverWithSSGAExpansion( + pomdp, lookAhead, 2, rounds, backups); + + solver.solve(); + + CyberDeceptionSimulation cyberDecSim = + new CyberDeceptionSimulation(solver, simLength, envIP, envPort); + + cyberDecSim.runSimulation(); + cyberDecSim.logToFile(storageDir + "/" + "interaction.json"); + cyberDecSim.writeDotFile(storageDir, "interaction"); + + } + + /* for IPOMDPs */ + else if (line.hasOption("i")) { + + LOGGER.info("Simulating IPOMDP..."); + int simLength = new Integer(line.getOptionValue("x")); + + LOGGER.info("Starting simulation"); + + /* initialize IPOMDP */ + IPOMDPParser parser = new IPOMDPParser(domainFile); + parser.parseDomain(); + + IPOMDP ipomdp = new IPOMDP(parser, lookAhead, simLength * 2); + + for (BaseSolver solver : ipomdp.lowerLevelSolutions) { + + /* set context */ + solver.f.setGlobals(); + + /* make policy graph */ + PolicyGraph pg = new PolicyGraph((OfflinePBVISolver) solver); + pg.makeGraph(); + + /* store policy graph solution */ + pg.writeDotFile( + storageDir, + "policy_graph_frame_" + pg.solver.f.frameID); + + pg.writeJSONFile( + storageDir, + "policy_graph_frame_" + pg.solver.f.frameID); + + /* make conditional plan */ + ConditionalPlanTree T = new ConditionalPlanTree(solver, simLength); + T.buildTree(); + + /* Store conditional Plan */ + T.writeDotFile( + storageDir, + "plan_frame_" + T.f.frameID); + + T.writeJSONFile( + storageDir, + "plan_frame_" + T.f.frameID); + } + + ipomdp.clearLowerLevelSolutions(); + + /* set context back to IPOMDP */ + ipomdp.setGlobals(); + + OnlineInteractiveSymbolicPerseus solver = + new OnlineInteractiveSymbolicPerseus( + ipomdp, + new FullBeliefExpansion(ipomdp), 1, backups); + + CyberDeceptionSimulation cyberDecSim = + new CyberDeceptionSimulation(solver, simLength, envIP, envPort); + cyberDecSim.runSimulation(); + + cyberDecSim.logToFile(storageDir + "/" + "interaction.json"); + cyberDecSim.writeDotFile(storageDir, "interaction"); + } + + } + + catch (Exception e) { + System.out.println("While parsing args: " + e.getMessage()); + Executable.printHelp(opt); + e.printStackTrace(); + System.exit(-1); + } + + } +} diff --git a/Protos/src/thinclab/executables/IPOMDPSolver.java b/Protos/src/thinclab/executables/IPOMDPSolver.java index 6bb396c8..2322adc5 100644 --- a/Protos/src/thinclab/executables/IPOMDPSolver.java +++ b/Protos/src/thinclab/executables/IPOMDPSolver.java @@ -174,7 +174,7 @@ public static void main(String[] args) { solver.makeConditionalPlan(planDir); } - /* conditional plan and policy graph */ + /* sim */ if (line.hasOption("t")) { int simIters = Integer.parseInt(line.getOptionValue("t")); solver.initializeIPOMDP(simIters * 2); diff --git a/Protos/src/thinclab/executables/RunSimulations.java b/Protos/src/thinclab/executables/RunSimulations.java index b4e05b60..445dbebf 100644 --- a/Protos/src/thinclab/executables/RunSimulations.java +++ b/Protos/src/thinclab/executables/RunSimulations.java @@ -209,7 +209,7 @@ else if (line.hasOption("i")) { StochasticSimulation ss = new StochasticSimulation(solver, simLength); ss.runSimulation(); - ss.logToFile(storageDir + "/" + "sim" + i + ".csv"); + ss.logToFile(storageDir + "/" + "sim" + i + ".json"); ss.writeDotFile(storageDir, "sim" + i); } } diff --git a/Protos/src/thinclab/simulations/CyberDeceptionEnvironmentConnector.java b/Protos/src/thinclab/simulations/CyberDeceptionEnvironmentConnector.java new file mode 100644 index 00000000..bd1c30e2 --- /dev/null +++ b/Protos/src/thinclab/simulations/CyberDeceptionEnvironmentConnector.java @@ -0,0 +1,111 @@ +/* + * THINC Lab at UGA | Cyber Deception Group + * + * Author: Aditya Shinde + * + * email: shinde.aditya386@gmail.com + */ +package thinclab.simulations; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.PrintWriter; +import java.util.concurrent.TimeUnit; + +import org.apache.log4j.Logger; + +/* + * @author adityas + * + */ +public class CyberDeceptionEnvironmentConnector extends NetworkedEnvironmentConnector { + + /* + * Defines the network connector for the cyber deception domain + */ + + private static final Logger LOGGER = + Logger.getLogger(CyberDeceptionEnvironmentConnector.class); + + // ----------------------------------------------------------------------------------- + + public CyberDeceptionEnvironmentConnector(String envIP, int envPort) { + + super(envIP, envPort); + LOGGER.info("Initialized CyberDeceptionEnvironmentConnector"); + } + + // ----------------------------------------------------------------------------------- + + @Override + public boolean syncEstablished(BufferedReader envInStream) { + + String hello = ""; + + try { + hello = envInStream.readLine().trim(); + } + + catch (IOException e) { + LOGGER.error("While reading from socket's instream: " + e.getMessage()); + e.printStackTrace(); + System.exit(-1); + } + + /* defender agent should try to handshake with a 'hello' */ + if (hello.contains("hello")) return true; + else { + LOGGER.debug("Expected 'hello', got " + hello); + return false; + } + } + + @Override + public void waitForAck(BufferedReader envInStream) { + + while (true) { + + String ack = ""; + + try { + ack = envInStream.readLine().trim(); + } + + catch (Exception e) { + LOGGER.error("While reading stream for ack: " + e.getMessage()); + e.printStackTrace(); + System.exit(-1); + } + + if (ack.contains("done")) break; + + else { + + LOGGER.warn("Did not get ack"); + LOGGER.debug("Retrying in 1 second"); + + try { + TimeUnit.SECONDS.sleep(1); + } + + catch (InterruptedException e) { + LOGGER.error("While sleeping"); + e.printStackTrace(); + System.exit(-1); + } + } + } + + } + + @Override + public void sendAck(PrintWriter envOutStream) { + envOutStream.println("done"); + } + + @Override + public String[] handleObservation(String obs) { + return obs.split(";"); + } + +} diff --git a/Protos/src/thinclab/simulations/CyberDeceptionSimulation.java b/Protos/src/thinclab/simulations/CyberDeceptionSimulation.java new file mode 100644 index 00000000..631bf6bf --- /dev/null +++ b/Protos/src/thinclab/simulations/CyberDeceptionSimulation.java @@ -0,0 +1,57 @@ +/* + * THINC Lab at UGA | Cyber Deception Group + * + * Author: Aditya Shinde + * + * email: shinde.aditya386@gmail.com + */ +package thinclab.simulations; + +import org.apache.log4j.Logger; + +import thinclab.decisionprocesses.DecisionProcess; +import thinclab.legacy.DD; +import thinclab.solvers.BaseSolver; + +/* + * @author adityas + * + */ +public class CyberDeceptionSimulation extends StochasticSimulation { + + /* + * Provides decision making interface for an actual agent + * + * The interaction takes place over HTTP + */ + + private CyberDeceptionEnvironmentConnector envConnector; + + private static final long serialVersionUID = 9022031217500704107L; + private static final Logger LOGGER = Logger.getLogger(CyberDeceptionSimulation.class); + + public CyberDeceptionSimulation( + BaseSolver solver, + int iterations, + String envIP, + int envPort) { + + super(solver, iterations); + this.envConnector = new CyberDeceptionEnvironmentConnector(envIP, envPort); + + LOGGER.info("Starting agent based simulation"); + } + + @Override + public String[] act(DecisionProcess DP, DD belief, String action) { + /* + * Override to send actions to an actual agent and get observations + */ + + LOGGER.info("Sending action " + action + " to the environment connector"); + String[] obs = this.envConnector.step(action); + + return obs; + } + +} diff --git a/Protos/src/thinclab/simulations/ExternalEnvironmentConnector.java b/Protos/src/thinclab/simulations/ExternalEnvironmentConnector.java new file mode 100644 index 00000000..aaf895ac --- /dev/null +++ b/Protos/src/thinclab/simulations/ExternalEnvironmentConnector.java @@ -0,0 +1,25 @@ +/* + * THINC Lab at UGA | Cyber Deception Group + * + * Author: Aditya Shinde + * + * email: shinde.aditya386@gmail.com + */ +package thinclab.simulations; + +/* + * @author adityas + * + */ +public interface ExternalEnvironmentConnector { + + /* + * Provides an interface for agent based simulations to interact with external + * environments + */ + + public void establishSync(); + public void sendAction(String action); + public String[] getObservation(); + +} diff --git a/Protos/src/thinclab/simulations/NetworkedEnvironmentConnector.java b/Protos/src/thinclab/simulations/NetworkedEnvironmentConnector.java new file mode 100644 index 00000000..c2c5b845 --- /dev/null +++ b/Protos/src/thinclab/simulations/NetworkedEnvironmentConnector.java @@ -0,0 +1,177 @@ +/* + * THINC Lab at UGA | Cyber Deception Group + * + * Author: Aditya Shinde + * + * email: shinde.aditya386@gmail.com + */ +package thinclab.simulations; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.PrintWriter; +import java.net.Socket; +import java.util.concurrent.TimeUnit; + +import org.apache.log4j.Logger; + +import cern.colt.Arrays; + +/* + * @author adityas + * + */ +abstract public class NetworkedEnvironmentConnector implements ExternalEnvironmentConnector { + + /* + * Handles interactions with an environment accessible over the network + * + */ + + private String envIP; + private int envPort; + + private Socket envConnectorSocket; + private PrintWriter envOutStream; + private BufferedReader envInStream; + + private static final Logger LOGGER = Logger.getLogger(NetworkedEnvironmentConnector.class); + + // ------------------------------------------------------------------------------ + + public NetworkedEnvironmentConnector(String envIP, int envPort) { + + this.envIP = envIP; + this.envPort = envPort; + } + + // ------------------------------------------------------------------------------ + + private void createEnvStreams() throws Exception { + /* + * Connects to the environment and returns the socket + */ + + this.envConnectorSocket = new Socket(this.envIP, this.envPort); + this.envOutStream = new PrintWriter(this.envConnectorSocket.getOutputStream(), true); + this.envInStream = + new BufferedReader( + new InputStreamReader(this.envConnectorSocket.getInputStream())); + + LOGGER.debug("Connected to " + envIP + " on port " + envPort); + } + + // ------------------------------------------------------------------------------ + + @Override + public void establishSync() { + /* + * Handshakes with the server over the network and populates the input and + * output streams + */ + + /* try connecting to server */ + try { + + this.createEnvStreams(); + } + + catch (Exception e) { + LOGGER.error("While connecting to server: " + e.getMessage()); + e.printStackTrace(); + System.exit(-1); + } + + /* exchange init messages or whatever to complete sync */ + while (true) { + + if (this.syncEstablished(this.envInStream)) { + LOGGER.info("Ready to interact with env on " + envIP); + break; + } + + else { + + LOGGER.warn("Could not establish sync with the env."); + LOGGER.debug("Retrying in 5 seconds"); + + try { + TimeUnit.SECONDS.sleep(5); + } + + catch (InterruptedException e) { + LOGGER.error("While sleeping"); + e.printStackTrace(); + System.exit(-1); + } + } + } + + } + + @Override + public void sendAction(String action) { + + this.envOutStream.println(action); + this.waitForAck(this.envInStream); + LOGGER.info("Sent action " + action); + } + + @Override + public String[] getObservation() { + + String[] observation = null; + + try { + String obs = this.envInStream.readLine().trim(); + + /* create observation array */ + observation = this.handleObservation(obs); + LOGGER.info("Got observation " + Arrays.toString(observation)); + + /* send ack */ + this.sendAck(this.envOutStream); + } + + catch (IOException e) { + LOGGER.error("While getting observations from env: " + e.getMessage()); + e.printStackTrace(); + System.exit(-1); + } + + if (observation == null) LOGGER.error("observation is null"); + + return observation; + } + + // --------------------------------------------------------------------------------- + + abstract public boolean syncEstablished(BufferedReader envInStream); + abstract public void waitForAck(BufferedReader envInStream); + abstract public void sendAck(PrintWriter envOutStream); + abstract public String[] handleObservation(String obs); + + public String[] step(String action) { + + /* handshake with the env server if required */ + this.establishSync(); + + /* send action */ + this.sendAction(action); + + /* get observation */ + String[] observation = this.getObservation(); + + /* close the connection */ + try { envConnectorSocket.close(); } + catch (Exception e) { + LOGGER.error("While closing the socket: " + e.getMessage()); + e.printStackTrace(); + System.exit(-1); + } + + return observation; + } + +} diff --git a/Protos/src/thinclab/simulations/Simulation.java b/Protos/src/thinclab/simulations/Simulation.java index 124b81ba..5b699345 100644 --- a/Protos/src/thinclab/simulations/Simulation.java +++ b/Protos/src/thinclab/simulations/Simulation.java @@ -93,7 +93,7 @@ public int step(BaseSolver solver, int currentNode) { String action = solver.getActionForBelief(currentBelief); this.idToNodeMap.get(currentNode).actName = action; - String[] obs = this.sampleObservation(solver.f, currentBelief, action); + String[] obs = this.act(solver.f, currentBelief, action); /* record the step */ this.beliefSequence.add(solver.f.getBeliefString(currentBelief)); @@ -160,7 +160,7 @@ public int step(BaseSolver solver, int currentNode) { } } - public String[] sampleObservation(DecisionProcess DP, DD belief, String action) { + public String[] act(DecisionProcess DP, DD belief, String action) { /* * Sample observation based on observation probabilities */ diff --git a/Protos/src/thinclab/solvers/OnlineInteractiveSymbolicPerseus.java b/Protos/src/thinclab/solvers/OnlineInteractiveSymbolicPerseus.java index 91c25b45..c28f7288 100644 --- a/Protos/src/thinclab/solvers/OnlineInteractiveSymbolicPerseus.java +++ b/Protos/src/thinclab/solvers/OnlineInteractiveSymbolicPerseus.java @@ -344,7 +344,8 @@ public void boundedPerseusStartFromCurrent( logger.info("I: " + stepId + " \tB ERROR: " + String.format(Locale.US, "%.03f", bellmanErr) + " \tUSED/TOTAL BELIEFS: " + numIter + "/" + beliefRegion.length - + " \tA VECTORS: " + this.alphaVectors.length); + + " \tA VECTORS: " + this.alphaVectors.length + + " \tAPPROX. CONV PATIENCE: " + this.numSimilar); /* report diagnostics on exec times */ Diagnostics.reportDiagnostics(); @@ -361,13 +362,13 @@ public void boundedPerseusStartFromCurrent( break; } - if (stepId > 20 && errorVar < 0.0001) { + if (stepId > 20 && errorVar < 0.00001) { logger.warn("DECLARING APPROXIMATE CONVERGENCE AT ERROR: " + bellmanErr + " BECAUSE OF LOW ERROR VARIANCE " + errorVar); break; } - if (stepId > 10 && this.declareApproxConvergenceForAlphaVectors( + if (stepId > 20 && this.declareApproxConvergenceForAlphaVectors( this.alphaVectors.length, numIter, beliefRegion.length)) { logger.warn("DECLARING APPROXIMATE CONVERGENCE AT ERROR: " + bellmanErr + " BECAUSE ALL BELIEFS ARE BEING USED AND NUM ALPHAS IS CONSTANT"); diff --git a/Protos/src/thinclab/solvers/OnlineSolver.java b/Protos/src/thinclab/solvers/OnlineSolver.java index dce9bb5d..c23cd0a0 100644 --- a/Protos/src/thinclab/solvers/OnlineSolver.java +++ b/Protos/src/thinclab/solvers/OnlineSolver.java @@ -173,7 +173,7 @@ public boolean declareApproxConvergenceForAlphaVectors( this.numSimilar += 1; /* set all trackers to initial values if declaring convergence */ - if (this.numSimilar >= 5) { + if (this.numSimilar >= 9) { this.numAlphas = -1; this.numBeliefs = -1;