fm_agent: Wait for fastmodel subprocess to terminate when resetting or terminating it

fm_agent/fm_agent.py

@@ -16,8 +16,8 @@
 limitations under the License.
 """
 
+import atexit
 import multiprocessing
-import sys
 import os
 from subprocess import Popen
 import time
@@ -86,10 +86,12 @@ def __init__(self, model_name=None, model_config=None, logger=None, enable_gdbse
             @param model_config is the config file to the fast model
         """
 
+        atexit.register(self.__del__)
+
         self.fastmodel_name = model_name
         self.config_name    = model_config
         self.enable_gdbserver = enable_gdbserver
-        self.subprocess = None
+        self.subprocess:Popen = None
 
         #If logging not provided, use default log
         if logger:
@@ -102,15 +104,19 @@ def __init__(self, model_name=None, model_config=None, logger=None, enable_gdbse
         self.socket = None # running instant of socket
         self.configuration = FastmodelConfig()
 
+        self.IRIS_port = None
+        self.terminal_ports = [None]*NUM_FVP_UART
+
         if model_config:
             self.setup_simulator(model_name,model_config)
         else:
             pass
 
     def __del__(self):
-        if isinstance(self.subprocess, Popen):
-            self.subprocess.terminate()
-            self.subprocess.wait()
+        self.__closeConnection()
+        self.__releaseModel()
+        self.__terminateSubprocess()
+        atexit.unregister(self.__del__)
 
     def setup_simulator(self, model_name, model_config):
         """ setup the simulator, this is crucial before you can start a simulator.
@@ -190,24 +196,37 @@ def is_simulator_alive(self):
         """return if the terminal socket is connected"""
         return bool(self.model)
 
-    def start_simulator(self, stream=sys.stdout):
+    def start_simulator(self):
         """ launch given fastmodel with configs """
         if check_import():
-            import iris.debug
-            self.subprocess, IRIS_port, outs = launch_FVP_IRIS(self.model_binary, self.model_config_file, self.model_options)
-            if stream:
-                print(outs, file=stream)
-            self.model = iris.debug.NetworkModel('localhost',IRIS_port)
-            # check which host socket port is used for terminal0
-            terminal = self.model.get_target(self.model_terminal)
-            self.port = terminal.read_register('Default.Port')
-            self.host = "localhost"
+            self.__spawn_simulator()
+
             self.image = None
 
             return True
         else:
             raise SimulatorError("fastmodel product was NOT installed correctly")
 
+    def __spawn_simulator(self):
+        import iris.debug
+
+        self.host = "localhost"
+
+        self.subprocess, self.IRIS_port, self.terminal_ports = launch_FVP_IRIS(
+            self.model_binary, self.model_config_file, self.model_options)
+
+        self.model = iris.debug.NetworkModel(self.host,self.IRIS_port)
+
+        terminal = self.model.get_target(self.model_terminal)
+
+        if self.terminal_ports[0] is None:
+            # This can be incorrect when the port is set explicitly as the FVP can choose a different port without
+            # updating Default.Port.
+            self.port = terminal.read_register('Default.Port')
+            self.logger.prn_wrn(f'Could not get port for telnetterminal0 from FVP output, best guess is {self.port}')
+        else:
+            self.port = self.terminal_ports[0]
+
     def load_simulator(self,image):
         """ Load a launched fastmodel with given image(full path)"""
         if self.is_simulator_alive():
@@ -241,22 +260,16 @@ def reset_simulator(self):
         if self.is_simulator_alive():
             self.logger.prn_wrn("STOP and RESTART FastModel")
             self.__closeConnection()
-            self.model.release(shutdown=True)
-            time.sleep(1)
-            import iris.debug
+            self.__releaseModel()
+            self.__terminateSubprocess()
+
+            self.__spawn_simulator()
 
-            self.subprocess, IRIS_port, outs = launch_FVP_IRIS(self.model_binary, self.model_config_file)
-            if IRIS_port==0:
-                print(outs)
-                return False
-            self.model = iris.debug.NetworkModel('localhost',IRIS_port)
-            # check which host socket port is used for terminal0
-            terminal = self.model.get_target(self.model_terminal)
-            self.port = terminal.read_register('Default.Port')
-            cpu = self.model.get_cpus()[0]
             if self.image:
+                cpu = self.model.get_cpus()[0]
                 cpu.load_application(self.image)
                 self.logger.prn_wrn("RELOAD new image to FastModel")
+
             self.model.run(blocking=False)
             self.__connect_terminal()
             self.logger.prn_wrn("Reconnect Terminal")
@@ -326,8 +339,6 @@ def __closeConnection(self):
             self.socket.close()
             self.logger.prn_inf("Closing terminal socket connection")
             self.socket = None
-        else:
-            self.logger.prn_inf("Terminal socket connection already closed")
 
     def __run_to_breakpoint(self):
         try:
@@ -415,12 +426,26 @@ def shutdown_simulator(self):
                 self.__CodeCoverage()
             self.logger.prn_inf("Fast-Model agent shutting down model")
             self.__closeConnection()
-            self.model.release(shutdown=True)
-            self.model=None
-            time.sleep(1)
+            self.__releaseModel()
+            self.__terminateSubprocess()
         else:
             self.logger.prn_inf("Model already shutdown")
 
+    def __releaseModel(self):
+        if self.model:
+            self.model.release(shutdown=True)
+            del self.model
+            self.model = None
+
+    def __terminateSubprocess(self):
+        if self.subprocess:
+            self.subprocess.terminate()
+            if self.subprocess.wait(3) is None:
+                self.subprocess.kill()
+                self.subprocess.wait()
+                del self.subprocess
+                self.subprocess = None
+
     def list_avaliable_models(self):
         """ return a dictionary of models and configs """
         return self.configuration.get_all_configs()

fm_agent/utils.py

@@ -22,10 +22,10 @@
 from functools import partial
 import subprocess
 from subprocess import Popen, PIPE, STDOUT
-from threading  import Thread
-from queue import Queue, Empty
+
 ON_POSIX = 'posix' in sys.builtin_module_names
 
+NUM_FVP_UART = 5
 
 class SimulatorError(Exception):
     """
@@ -119,38 +119,32 @@ def remove_gcda(rootdir="."):
             if file.endswith(".gcda"):
                 os.remove(os.path.join(root, file))
 
-def enqueue_output(out, queue):
-    for line in iter(out.readline, b''):
-        queue.put(line)
-    out.close()
-
 def launch_FVP_IRIS(model_exec, config_file='', model_options=[]):
     """Launch FVP with IRIS Server listening"""
     cmd_line = [model_exec, '-I', '-p']
     cmd_line.extend(model_options)
     if config_file:
         cmd_line.extend(['-f' , config_file])
     logging.info(cmd_line)
-    fm_proc = Popen(cmd_line,stdout=PIPE,stderr=STDOUT, close_fds=ON_POSIX)
-    out_q = Queue()
-    reader_t = Thread(target=enqueue_output, args=(fm_proc.stdout, out_q))
-    reader_t.daemon = True
-    reader_t.start()
-
-    stdout=''
-    port = 0
-    end = False
-
-    while not end:
-        try: line = out_q.get(timeout=1).decode().strip()
-        except Empty:
-            end = True
-        else:
-            if line.startswith("Iris server started listening to port"):
-                port = int(line[-5:])
-            stdout = stdout + line + "\n"
 
-    return (fm_proc, port, stdout)
+    fm_proc = Popen(cmd_line, stdout=PIPE, stderr=STDOUT, close_fds=ON_POSIX)
+
+    # Read Iris and telnet terminal ports from stdin. Stop on the first blank line.
+    iris_port = None
+    terminal_ports = [None]*NUM_FVP_UART
+    for line in fm_proc.stdout:
+        line = line.strip().decode('utf-8')
+        if line == '':
+            break
+
+        words = line.split()
+        if words[0].startswith('telnetterminal'):
+            which = int(words[0][-2])
+            terminal_ports[which] = int(words[-1])
+        elif words[0].startswith('Iris'):
+            iris_port = int(words[-1])
+
+    return (fm_proc, iris_port, terminal_ports)
 
 def getenv_replace(s):
     """Replace substrings enclosed by {{ and }} with values from the environment so that e.g. '{{USER}}' becomes 'root'.