[microTVM] Zephyr: RISCV support for Zephyr QEMU RISCV-32/64 (#7804)

* working on qemu

* debugging

* riscv hacks

* config added

* change target platforms

* fix merge

* debugging issue with zephyr 2.5

* cleanup

* working on qemu

* debugging

* riscv hacks

* config added

* change target platforms

* fix merge

* debugging issue with zephyr 2.5

* cleanup

* testing

* pass riscv64

* fix merge

* small fix

* update vm_name

* add zephyr version

* add comment for riscv32 issue

* remove debug messages

* cleanup

* cleanup

* change workspace

* fix zephyr version

* cleanup

* change to symlink

* fix flag

* add comment

* lint check

* lint fix

* fix format

* rename debugger

* rework args

Co-authored-by: Mehrdad Hessar <mhessar@octoml.local>
Co-authored-by: Andrew Reusch <areusch@octoml.ai>

apps/microtvm/reference-vm/base-box-tool.py

@@ -358,14 +358,16 @@ def test_command(args):
 
 
 def release_command(args):
+    vm_name = f"tlcpack/microtvm-{args.platform}-{args.platform_version}"
+
     if not args.skip_creating_release_version:
         subprocess.check_call(
             [
                 "vagrant",
                 "cloud",
                 "version",
                 "create",
-                f"tlcpack/microtvm-{args.platform}",
+                vm_name,
                 args.release_version,
             ]
         )
@@ -379,7 +381,7 @@ def release_command(args):
                 "cloud",
                 "publish",
                 "-f",
-                f"tlcpack/microtvm-{args.platform}",
+                vm_name,
                 args.release_version,
                 provider_name,
                 os.path.join(
@@ -392,23 +394,11 @@ def release_command(args):
         )
 
 
-ALL_COMMANDS = {
-    "build": build_command,
-    "test": test_command,
-    "release": release_command,
-}
-
-
 def parse_args():
     parser = argparse.ArgumentParser(
         description="Automates building, testing, and releasing a base box"
     )
-    parser.add_argument(
-        "command",
-        default=",".join(ALL_COMMANDS),
-        choices=ALL_COMMANDS,
-        help="Action or actions (comma-separated) to perform.",
-    )
+    subparsers = parser.add_subparsers(help="Action to perform.")
     parser.add_argument(
         "platform",
         help="Name of the platform VM to act on. Must be a sub-directory of this directory.",
@@ -417,70 +407,70 @@ def parse_args():
         "--provider",
         choices=ALL_PROVIDERS,
         action="append",
-        default=[],
-        help="Name of the provider or providers to act on; if not specified, act on all",
+        default=list(ALL_PROVIDERS),
+        help="Name of the provider or providers to act on; if not specified, act on all.",
     )
-    parser.add_argument(
+
+    parser_build = subparsers.add_parser("build", help="Build a base box.")
+    parser_build.set_defaults(func=build_command)
+    parser_test = subparsers.add_parser("test", help="Test a base box before release.")
+    parser_test.set_defaults(func=test_command)
+    parser_release = subparsers.add_parser("release", help="Release base box to cloud.")
+    parser_release.set_defaults(func=release_command)
+
+    parser_build.add_argument(
+        "--debug-packer",
+        action="store_true",
+        help=("Run packer in debug mode, and write log to the base-box directory."),
+    )
+    parser_test.add_argument(
         "--skip-build",
         action="store_true",
         help=(
-            "For use with the 'test' command. If given, assume a box has already been built in "
+            "If given, assume a box has already been built in "
             "the release-test subdirectory. Attach a USB device to this box and execute the "
             "release test script--do not delete it."
         ),
     )
-    parser.add_argument(
+    parser_test.add_argument(
         "--test-device-serial",
         help=(
             "If given, attach the test device with this USB serial number. Corresponds to the "
             "iSerial field from `lsusb -v` output."
         ),
     )
-    parser.add_argument(
+    parser_test.add_argument(
+        "--microtvm-platform",
+        choices=ALL_MICROTVM_PLATFORMS,
+        required=True,
+        help="MicroTVM platfrom used for testing.",
+    )
+    parser_release.add_argument(
         "--release-version",
+        required=True,
         help="Version to release, in the form 'x.y.z'. Must be specified with release.",
     )
-    parser.add_argument(
+    parser_release.add_argument(
         "--skip-creating-release-version",
         action="store_true",
-        help="With release, skip creating the version and just upload for this provider.",
-    )
-    parser.add_argument(
-        "--debug-packer",
-        action="store_true",
-        help=(
-            "When the build command is given, run packer in debug mode, and write log to the "
-            "base-box directory"
-        ),
+        help="Skip creating the version and just upload for this provider.",
     )
-
-    parser.add_argument(
-        "--microtvm-platform",
-        default="stm32f746xx",
-        choices=ALL_MICROTVM_PLATFORMS,
-        help="For use with 'test' command. MicroTVM platfrom that are used for testing.",
+    parser_release.add_argument(
+        "--platform-version",
+        required=True,
+        help="Platform version to release, in the form 'x.y'.",
     )
 
     return parser.parse_args()
 
 
 def main():
     args = parse_args()
+
     if os.path.sep in args.platform or not os.path.isdir(os.path.join(THIS_DIR, args.platform)):
         sys.exit(f"<platform> must be a sub-direcotry of {THIS_DIR}; got {args.platform}")
 
-    if not args.provider:
-        args.provider = list(ALL_PROVIDERS)
-
-    todo = []
-    for phase in args.command.split(","):
-        if phase not in ALL_COMMANDS:
-            sys.exit(f"unknown command: {phase}")
-
-        todo.append(ALL_COMMANDS[phase])
-
-    for phase in todo:
-        phase(args)
+    args.func(args)
 
 
 if __name__ == "__main__":

apps/microtvm/reference-vm/zephyr/base-box/setup.sh

@@ -94,7 +94,7 @@ wget --no-verbose -O $ZEPHYR_SDK_FILE \
     https://github.com/zephyrproject-rtos/sdk-ng/releases/download/v${ZEPHYR_SDK_VERSION}/zephyr-sdk-${ZEPHYR_SDK_VERSION}-x86_64-linux-setup.run
 chmod +x $ZEPHYR_SDK_FILE
 "./$ZEPHYR_SDK_FILE" -- -d ~/zephyr-sdk -y
-rm -rf ZEPHYR_SDK_FILE
+rm -rf "${ZEPHYR_SDK_FILE}"
 
 # GDB for Zephyr SDK depends on python3.8
 sudo add-apt-repository ppa:deadsnakes/ppa

apps/microtvm/zephyr/demo_runtime/boards/qemu_riscv32.conf

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This file is specific to the QEMU-emulated RISCV32 microTVM board.
+
+# For TVMPlatformGenerateRandom(). Remember, these values do not need to be truly random.
+CONFIG_TEST_RANDOM_GENERATOR=y
+CONFIG_TIMER_RANDOM_GENERATOR=y
+
+# Default is 512, raised here for operations with large floating point data.
+CONFIG_MAIN_STACK_SIZE=2048
+
+# For floating point operations. It has exception on floating point operations
+# without this flag.
+CONFIG_FPU_SHARING=y

apps/microtvm/zephyr/demo_runtime/boards/qemu_riscv64.conf

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This file is specific to the QEMU-emulated RISCV64 microTVM board.
+
+# For TVMPlatformGenerateRandom(). Remember, these values do not need to be truly random.
+CONFIG_TEST_RANDOM_GENERATOR=y
+CONFIG_TIMER_RANDOM_GENERATOR=y
+
+# Default 512, for operations with large floating point data. 
+CONFIG_MAIN_STACK_SIZE=2048

apps/microtvm/zephyr/demo_runtime/qemu-hack/qemu-system-i386

@@ -30,4 +30,9 @@ while [ "$#" -gt 0 ]; do
     shift
 done
 
+# For debugging
+if [ "${TVM_QEMU_DEBUG}" != "" ]; then
+    ARGS=( "${ARGS[@]}" -s -S )
+fi
+
 "${ARGS[@]}"

apps/microtvm/zephyr/demo_runtime/qemu-hack/qemu-system-riscv32

@@ -0,0 +1 @@
+qemu-system-i386

apps/microtvm/zephyr/demo_runtime/qemu-hack/qemu-system-riscv64

@@ -0,0 +1 @@
+qemu-system-i386

apps/microtvm/zephyr/demo_runtime/src/main.c

@@ -265,6 +265,12 @@ static uint8_t main_rx_buf[RING_BUF_SIZE_BYTES];
 // The main function of this application.
 extern void __stdout_hook_install(int (*hook)(int));
 void main(void) {
+  // TODO (mehrdadh): Update this when zephyr version has updated to 2.6.
+  // Update zephyr to latest version to use with qemu_riscv32.
+#ifdef CONFIG_BOARD_QEMU_RISCV32
+  k_float_enable(_current, 0);
+#endif
+
 #ifdef CONFIG_LED
   int ret;
   led0_pin = device_get_binding(LED0);

python/tvm/micro/contrib/zephyr.py

@@ -18,9 +18,11 @@
 """Defines a compiler integration that uses an externally-supplied Zephyr project."""
 
 import collections
+import copy
 import logging
 import multiprocessing
 import os
+import pathlib
 import re
 import tempfile
 import textwrap
@@ -428,12 +430,28 @@ def _get_device_args(self, cmake_entries):
             f"runner {flash_runner}"
         )
 
+    def _zephyr_transport(self, micro_binary):
+        qemu_debugger = None
+        if self._debug_rpc_session:
+            qemu_debugger = debugger.RpcDebugger(
+                self._debug_rpc_session,
+                debugger.DebuggerFactory(
+                    QemuGdbDebugger,
+                    (micro_binary.abspath(micro_binary.debug_files[0]),),
+                    {},
+                ),
+            )
+
+        return ZephyrQemuTransport(
+            micro_binary.base_dir, startup_timeout_sec=30.0, qemu_debugger=qemu_debugger
+        )
+
     def flash(self, micro_binary):
         cmake_entries = read_cmake_cache(
             micro_binary.abspath(micro_binary.labelled_files["cmake_cache"][0])
         )
         if "qemu" in cmake_entries["BOARD"]:
-            return ZephyrQemuTransport(micro_binary.base_dir, startup_timeout_sec=30.0)
+            return self._zephyr_transport(micro_binary)
 
         build_dir = os.path.dirname(
             micro_binary.abspath(micro_binary.labelled_files["cmake_cache"][0])
@@ -532,6 +550,26 @@ def transport(self, micro_binary):
         )
 
 
+class QemuGdbDebugger(debugger.GdbDebugger):
+    def __init__(self, elf_file):
+        super(QemuGdbDebugger, self).__init__()
+        self._elf_file = elf_file
+
+    def popen_kwargs(self):
+        # expect self._elf file to follow the form .../zephyr/zephyr.elf
+        cmake_cache_path = pathlib.Path(self._elf_file).parent.parent / "CMakeCache.txt"
+        cmake_cache = read_cmake_cache(cmake_cache_path)
+        return {
+            "args": [
+                cmake_cache["CMAKE_GDB"],
+                "-ex",
+                "target remote localhost:1234",
+                "-ex",
+                f"file {self._elf_file}",
+            ],
+        }
+
+
 class QemuStartupFailureError(Exception):
     """Raised when the qemu pipe is not present within startup_timeout_sec."""
 
@@ -571,33 +609,47 @@ def write(self, data, timeout_sec):
 class ZephyrQemuTransport(Transport):
     """The user-facing Zephyr QEMU transport class."""
 
-    def __init__(self, base_dir, startup_timeout_sec=5.0, **kwargs):
+    def __init__(self, base_dir, startup_timeout_sec=5.0, qemu_debugger=None, **kwargs):
         self.base_dir = base_dir
         self.startup_timeout_sec = startup_timeout_sec
         self.kwargs = kwargs
         self.proc = None
         self.fd_transport = None
         self.pipe_dir = None
+        self.qemu_debugger = qemu_debugger
 
     def timeouts(self):
         return TransportTimeouts(
             session_start_retry_timeout_sec=2.0,
             session_start_timeout_sec=self.startup_timeout_sec,
-            session_established_timeout_sec=5.0,
+            session_established_timeout_sec=5.0 if self.qemu_debugger is None else 0,
         )
 
     def open(self):
         self.pipe_dir = tempfile.mkdtemp()
         self.pipe = os.path.join(self.pipe_dir, "fifo")
         self.write_pipe = os.path.join(self.pipe_dir, "fifo.in")
         self.read_pipe = os.path.join(self.pipe_dir, "fifo.out")
+
         os.mkfifo(self.write_pipe)
         os.mkfifo(self.read_pipe)
+        if self.qemu_debugger is not None:
+            if "env" in self.kwargs:
+                self.kwargs["env"] = copy.copy(self.kwargs["env"])
+            else:
+                self.kwargs["env"] = os.environ.copy()
+
+            self.kwargs["env"]["TVM_QEMU_DEBUG"] = "1"
+
         self.proc = subprocess.Popen(
             ["make", "run", f"QEMU_PIPE={self.pipe}"],
             cwd=self.base_dir,
             **self.kwargs,
         )
+
+        if self.qemu_debugger is not None:
+            self.qemu_debugger.start()
+
         # NOTE: although each pipe is unidirectional, open both as RDWR to work around a select
         # limitation on linux. Without this, non-blocking I/O can't use timeouts because named
         # FIFO are always considered ready to read when no one has opened them for writing.
@@ -612,6 +664,9 @@ def open(self):
         self.fd_transport.open()
 
     def close(self):
+        if self.qemu_debugger is not None:
+            self.qemu_debugger.stop()
+
         if self.fd_transport is not None:
             self.fd_transport.child_transport.write_monitor_quit()
             self.proc.wait()

tests/lint/check_file_type.py

@@ -131,11 +131,15 @@
     "tests/micro/zephyr/testdata/mnist-8.onnx",
     # microTVM Zephyr runtime
     "apps/microtvm/zephyr/demo_runtime/prj.conf",
+    "apps/microtvm/zephyr/demo_runtime/boards/qemu_x86.conf",
+    "apps/microtvm/zephyr/demo_runtime/boards/qemu_riscv32.conf",
+    "apps/microtvm/zephyr/demo_runtime/boards/qemu_riscv64.conf",
     "apps/microtvm/zephyr/demo_runtime/boards/nrf5340dk_nrf5340_cpuapp.conf",
     "apps/microtvm/zephyr/demo_runtime/boards/nucleo_f746zg.conf",
-    "apps/microtvm/zephyr/demo_runtime/boards/qemu_x86.conf",
     "apps/microtvm/zephyr/demo_runtime/boards/stm32f746g_disco.conf",
     "apps/microtvm/zephyr/demo_runtime/qemu-hack/qemu-system-i386",
+    "apps/microtvm/zephyr/demo_runtime/qemu-hack/qemu-system-riscv32",
+    "apps/microtvm/zephyr/demo_runtime/qemu-hack/qemu-system-riscv64",
     # microTVM Virtual Machines
     "apps/microtvm/reference-vm/zephyr/Vagrantfile",
     "apps/microtvm/reference-vm/zephyr/base-box/Vagrantfile.packer-template",