Create the benchmark runner

.gitignore

@@ -1,2 +1,4 @@
 *.idea*
 *__pycache__*
+venv
+.python-version

benchmark/reference_submissions/.gitignore

@@ -0,0 +1,3 @@
+*.zip*
+main.cpp
+

benchmark/runner/.gitignore

@@ -0,0 +1,3 @@
+.idea
+venv
+

benchmark/runner/README.md

@@ -0,0 +1,11 @@
+[[_TOC_]]
+
+## Energy Test
+
+### Power Board (LPM01A)
+![LPM01A Wiring](img/LPM01A.jpg)
+### Interface Board (STM32H573I-DK)
+![STM32H573I-DK Top Wiring](img/STM32H573I-DK-Top.jpg)
+![STM32H573I-DK Bottom Wiring](img/STM32H573I-DK-Bottom.png)
+### Device Under Test (L4R5ZI)
+![DUT Wiring](img/L4R5ZI.png)

benchmark/runner/device_under_test.py

@@ -0,0 +1,83 @@
+import re
+
+from interface_device import InterfaceDevice
+from serial_device import SerialDevice
+
+
+class DUT:
+  def __init__(self, port_device, baud_rate=115200):
+    interface = port_device
+    if not isinstance(port_device, InterfaceDevice):
+      interface = SerialDevice(port_device, baud_rate, "m-ready", '%')
+    self._port = interface
+    self._profile = None
+    self._model = None
+    self._name = None
+
+  def __enter__(self):
+    if isinstance(self._port, SerialDevice):
+      self._port.__enter__()
+    return self
+
+  def __exit__(self, *args):
+    if isinstance(self._port, SerialDevice):
+      self._port.__exit__(*args)
+
+  def _get_name(self):
+    for l in self._port.send_command("name"):
+      match = re.match(r'^m-(name)-dut-\[([^]]+)]$', l)
+      if match:
+        self.__setattr__(f"_{match.group(1)}", match.group(2))
+
+  def get_name(self):
+    if self._name is None:
+      self._get_name()
+    return self._name
+
+  def _get_profile(self):
+    for l in self._port.send_command("profile"):
+      match = re.match(r'^m-(model|profile)-\[([^]]+)]$', l)
+      if match:
+        self.__setattr__(f"_{match.group(1)}", match.group(2))
+
+  def get_model(self):
+    if self._model is None:
+      self._get_profile()
+    return self._model
+
+  def get_profile(self):
+    if self._profile is None:
+      self._get_profile()
+    return self._profile
+
+  def timestamp(self):
+    return self._port.send_command("timestamp")
+
+  def send_data(self, data):
+    size = len(data)
+    pass
+
+  def infer(self, number, warmups):
+    command = f"infer {number}"
+    if warmups:
+      command += f" {warmups}"
+    self._port.send_command(command)
+    self._port.send_command("results")
+
+  def get_help(self):
+    return self._port.send_command("help")
+
+  """
+  ULPMark for tinyML Firmware V0.0.1
+
+  help         : Print this information
+  name         : Print the name of the device
+  timestsamp   : Generate a timetsamp
+  db SUBCMD    : Manipulate a generic byte buffer
+    load N     : Allocate N bytes and set load counter
+    db HH[HH]* : Load 8-bit hex byte(s) until N bytes
+    print [N=16] [offset=0]
+               : Print N bytes at offset as hex
+  infer N [W=0]: Load input, execute N inferences after W warmup loops
+  results      : Return the result fp32 vector
+  """

benchmark/runner/interface_device.py

@@ -0,0 +1,3 @@
+class InterfaceDevice:
+  def send_command(self, command, end=None, echo=False):
+    pass

benchmark/runner/io_manager.py

@@ -0,0 +1,56 @@
+from interface_device import InterfaceDevice
+from serial_device import SerialDevice
+
+
+class IOManager(InterfaceDevice):
+  def __init__(self, port_device, baud_rate=115200):
+    self.port = SerialDevice(port_device, baud_rate, "m-ready", '%')
+
+  def __enter__(self):
+    self.port.__enter__()
+    return self
+
+  def __exit__(self, *args):
+    self.port.__exit__(*args)
+
+  def get_name(self):
+    return self.port.send_command("name")
+
+  def timestamp(self):
+    return self.port.send_command("timestamp")
+
+  def get_help(self):
+    return self.port.send_command("help")
+
+  def send_data(self, data):
+    size = len(data)
+    pass
+
+  def send_command(self, command, end=None, echo=False):
+    resp = self.port.send_command(f"dut {command}")
+    if len(resp) != 2 or resp[1] != "m-ready":
+      return None
+    resp = None
+    lines = []
+    while resp != 'm-ready':
+      resp = self.port.read_line()
+      resp = resp.replace("[dut]: ", "")
+      lines.append(resp)
+    return lines if len(lines) != 1 else lines[0]
+
+  """
+   help
+   name              device name
+   res               Reset polarity 0*,1
+   enable-timer      Enable timer ISR
+   disable-timer     Disable ISR
+   dut               DUT passthrough
+   i2c-enable        N : Enable I2C slave-mode to send N bytes (mod16)
+   i2c-disable       Disable above
+   load-vdata        Load 16 random bytes into vdata
+   show-vdata        Show vdata
+   et                Start Emon
+   tm                0=fall/1*=change
+   tp                [res]
+   version           firmware version
+  """

benchmark/runner/io_manager_enhanced.py

@@ -0,0 +1,16 @@
+from io_manager import IOManager
+
+
+class IOManagerEnhanced(IOManager):
+  def __init__(self, port_device, baud_rate=921600):
+    IOManager.__init__(self, port_device, baud_rate)
+
+  def get_files(self):
+    return self.port.send_command('ls')
+
+  def get_waves(self):
+    return [x for x in self.get_files() if x.lower().endswith("wav") or x == "m-ready"]
+
+  def play_wave(self, filename=None):
+    command = "play" + (f" {filename}" if filename else "")
+    return self.port.send_command(command)

benchmark/runner/main.py

@@ -0,0 +1,86 @@
+from device_under_test import DUT
+
+from io_manager_enhanced import IOManagerEnhanced
+from power_manager import PowerManager
+from io_manager import IOManager
+
+
+def test_power():
+  with PowerManager("/dev/tty.usbmodem204C355C55471") as power:
+      pass
+      for line in power.get_help():
+          print(line)
+      # power.acquire(500)
+
+
+def run_dut_test():
+  """
+  power on
+  m-ready
+  -mode energy
+  imit-done
+  m-ready
+  :return:
+  dut passthrough: profile
+  """
+
+def test_dut():
+    with DUT("/dev/tty.usbmodem14612303", 9600) as dut:
+        print(dut.get_name())
+        print(dut.get_model())
+        print(dut.get_profile())
+        print(dut.timestamp())
+        for l in dut.get_help():
+            print(l)
+
+
+def test_io_manager():
+  with IOManager('/dev/tty.usbmodem14612201') as io:
+      print(io.get_name())
+      with DUT(io) as dut:
+          print(dut.get_name())
+          # print(dut.get_model())
+          # print(dut.get_profile())
+          # print(dut.timestamp())
+          # for l in dut.get_help():
+          #     print(l)
+      for l in io.get_help():
+          print(l)
+
+
+def test_io_manager_enhanced():
+    with IOManagerEnhanced("/dev/tty.usbmodem146403") as io:
+        print(io.get_name())
+        with DUT(io) as dut:
+            print(dut.get_name())
+            print(dut.get_model())
+            print(dut.get_profile())
+            print(dut.timestamp())
+            for l in dut.get_help():
+                print(l)
+        print(io.get_waves())
+        print(io.play_wave())
+        # waves = io.list_waves()
+        # io.play_wave();
+        # for w in waves:
+        #     print(w)
+        #     # io.play_wave(w)
+
+
+if __name__ == '__main__':
+    # try:
+    #     test_io_manager()
+    # except:
+    #     pass
+    try:
+        test_io_manager_enhanced()
+    except:
+        pass
+    # try:
+    #     test_power()
+    # except:
+    #     pass
+    try:
+        test_dut()
+    except:
+        pass