From 83a32bdc24d4c1f8907d2c922eca8f55c690152e Mon Sep 17 00:00:00 2001 From: Mehrdad Hessar Date: Fri, 29 Jul 2022 07:20:55 -0700 Subject: [PATCH] [microTVM][Zephyr][projectAPI] Minimize project build commands (#12209) * move cmake args to generate project * remove zephyr board from flash and run --- .../template_project/CMakeLists.txt.template | 2 +- .../template_project/microtvm_api_server.py | 93 +++++++++++-------- .../how_to/work_with_microtvm/micro_tvmc.sh | 14 ++- tests/micro/common/test_tvmc.py | 44 ++++----- 4 files changed, 86 insertions(+), 67 deletions(-) diff --git a/apps/microtvm/zephyr/template_project/CMakeLists.txt.template b/apps/microtvm/zephyr/template_project/CMakeLists.txt.template index 710cf3550c0d..7f37efc599c2 100644 --- a/apps/microtvm/zephyr/template_project/CMakeLists.txt.template +++ b/apps/microtvm/zephyr/template_project/CMakeLists.txt.template @@ -23,7 +23,7 @@ set(ENV{QEMU_BIN_PATH} "${CMAKE_SOURCE_DIR}/qemu-hack") set(QEMU_PIPE "\${QEMU_PIPE}") # QEMU_PIPE is set by the calling TVM instance. -set(ENABLE_CMSIS ) + find_package(Zephyr HINTS $ENV{ZEPHYR_BASE}) project(microtvm_autogenerated_project) diff --git a/apps/microtvm/zephyr/template_project/microtvm_api_server.py b/apps/microtvm/zephyr/template_project/microtvm_api_server.py index 94bf51d1e551..81394f31ab18 100644 --- a/apps/microtvm/zephyr/template_project/microtvm_api_server.py +++ b/apps/microtvm/zephyr/template_project/microtvm_api_server.py @@ -61,6 +61,8 @@ BOARDS = API_SERVER_DIR / "boards.json" +CMAKELIST_FILENAME = "CMakeLists.txt" + # Used to check Zephyr version installed on the host. # We only check two levels of the version. ZEPHYR_VERSION = 2.7 @@ -275,13 +277,13 @@ def _get_nrf_device_args(options): ), server.ProjectOption( "verbose", - optional=["build"], + optional=["generate_project"], type="bool", help="Run build with verbose output.", ), server.ProjectOption( "west_cmd", - optional=["build"], + optional=["generate_project"], default=WEST_CMD, type="str", help=( @@ -292,14 +294,14 @@ def _get_nrf_device_args(options): server.ProjectOption( "zephyr_base", required=(["generate_project", "open_transport"] if not ZEPHYR_BASE else None), - optional=(["generate_project", "open_transport", "build"] if ZEPHYR_BASE else ["build"]), + optional=(["generate_project", "open_transport"] if ZEPHYR_BASE else ["build"]), default=ZEPHYR_BASE, type="str", help="Path to the zephyr base directory.", ), server.ProjectOption( "zephyr_board", - required=["generate_project", "build", "flash", "open_transport"], + required=["generate_project"], choices=list(BOARD_PROPERTIES), type="str", help="Name of the Zephyr board to build for.", @@ -419,7 +421,7 @@ def _create_prj_conf(self, project_dir, options): f.write("\n") API_SERVER_CRT_LIBS_TOKEN = "" - ENABLE_CMSIS_TOKEN = "" + CMAKE_ARGS_TOKEN = "" CRT_LIBS_BY_PROJECT_TYPE = { "host_driven": "microtvm_rpc_server microtvm_rpc_common aot_executor_module aot_executor common", @@ -456,6 +458,28 @@ def _cmsis_required(self, project_path: Union[str, pathlib.Path]) -> bool: return True return False + def _generate_cmake_args(self, mlf_extracted_path, options) -> str: + cmake_args = "\n# cmake args\n" + if options.get("verbose"): + cmake_args += "set(CMAKE_VERBOSE_MAKEFILE TRUE)\n" + + if options.get("zephyr_base"): + cmake_args += f"set(ZEPHYR_BASE {options['zephyr_base']})\n" + + if options.get("west_cmd"): + cmake_args += f"set(WEST {options['west_cmd']})\n" + + if self._is_qemu(options["zephyr_board"]): + # Some boards support more than one emulator, so ensure QEMU is set. + cmake_args += f"set(EMU_PLATFORM qemu)\n" + + cmake_args += f"set(BOARD {options['zephyr_board']})\n" + + enable_cmsis = self._cmsis_required(mlf_extracted_path) + cmake_args += f"set(ENABLE_CMSIS {str(enable_cmsis).upper()})\n" + + return cmake_args + def generate_project(self, model_library_format_path, standalone_crt_dir, project_dir, options): # Check Zephyr version version = self._get_platform_version(get_zephyr_base(options)) @@ -487,7 +511,7 @@ def generate_project(self, model_library_format_path, standalone_crt_dir, projec os.makedirs(extract_path) tf.extractall(path=extract_path) - if self._is_qemu(options): + if self._is_qemu(options["zephyr_board"]): shutil.copytree(API_SERVER_DIR / "qemu-hack", project_dir / "qemu-hack") # Populate CRT. @@ -502,16 +526,15 @@ def generate_project(self, model_library_format_path, standalone_crt_dir, projec shutil.copy2(src_path, dst_path) # Populate Makefile. - with open(project_dir / "CMakeLists.txt", "w") as cmake_f: - with open(API_SERVER_DIR / "CMakeLists.txt.template", "r") as cmake_template_f: + with open(project_dir / CMAKELIST_FILENAME, "w") as cmake_f: + with open(API_SERVER_DIR / f"{CMAKELIST_FILENAME}.template", "r") as cmake_template_f: for line in cmake_template_f: if self.API_SERVER_CRT_LIBS_TOKEN in line: crt_libs = self.CRT_LIBS_BY_PROJECT_TYPE[options["project_type"]] line = line.replace("", crt_libs) - if self.ENABLE_CMSIS_TOKEN in line: - enable_cmsis = self._cmsis_required(extract_path) - line = line.replace(self.ENABLE_CMSIS_TOKEN, str(enable_cmsis).upper()) + if self.CMAKE_ARGS_TOKEN in line: + line = self._generate_cmake_args(extract_path, options) cmake_f.write(line) @@ -541,23 +564,7 @@ def generate_project(self, model_library_format_path, standalone_crt_dir, projec def build(self, options): BUILD_DIR.mkdir() - cmake_args = ["cmake", ".."] - if options.get("verbose"): - cmake_args.append("-DCMAKE_VERBOSE_MAKEFILE:BOOL=TRUE") - - if options.get("zephyr_base"): - cmake_args.append(f"-DZEPHYR_BASE:STRING={options['zephyr_base']}") - - if options.get("west_cmd"): - cmake_args.append(f"-DWEST={options['west_cmd']}") - - if self._is_qemu(options): - # Some boards support more than one emulator, so ensure QEMU is set. - cmake_args.append(f"-DEMU_PLATFORM=qemu") - - cmake_args.append(f"-DBOARD:STRING={options['zephyr_board']}") - - check_call(cmake_args, cwd=BUILD_DIR) + check_call(["cmake", ".."], cwd=BUILD_DIR) args = ["make", "-j2"] if options.get("verbose"): @@ -570,22 +577,32 @@ def build(self, options): _KNOWN_QEMU_ZEPHYR_BOARDS = ("mps2_an521", "mps3_an547") @classmethod - def _is_qemu(cls, options): - return ( - "qemu" in options["zephyr_board"] - or options["zephyr_board"] in cls._KNOWN_QEMU_ZEPHYR_BOARDS - ) + def _is_qemu(cls, board: str) -> bool: + return "qemu" in board or board in cls._KNOWN_QEMU_ZEPHYR_BOARDS @classmethod def _has_fpu(cls, zephyr_board): fpu_boards = [name for name, board in BOARD_PROPERTIES.items() if board["fpu"]] return zephyr_board in fpu_boards + @classmethod + def _find_board_from_cmake_file(cls) -> str: + zephyr_board = None + with open(API_SERVER_DIR / CMAKELIST_FILENAME) as cmake_f: + for line in cmake_f: + if line.startswith("set(BOARD"): + zephyr_board = line.strip("\n").strip("set(BOARD ").strip(")") + break + + if not zephyr_board: + raise RuntimeError(f"No Zephyr board set in the {API_SERVER_DIR / CMAKELIST_FILENAME}.") + return zephyr_board + def flash(self, options): - if self._is_qemu(options): - return # NOTE: qemu requires no flash step--it is launched from open_transport. + zephyr_board = self._find_board_from_cmake_file() - zephyr_board = options["zephyr_board"] + if self._is_qemu(zephyr_board): + return # NOTE: qemu requires no flash step--it is launched from open_transport. # The nRF5340DK requires an additional `nrfjprog --recover` before each flash cycle. # This is because readback protection is enabled by default when this device is flashed. @@ -600,7 +617,9 @@ def flash(self, options): check_call(["make", "flash"], cwd=API_SERVER_DIR / "build") def open_transport(self, options): - if self._is_qemu(options): + zephyr_board = self._find_board_from_cmake_file() + + if self._is_qemu(zephyr_board): transport = ZephyrQemuTransport(options) else: transport = ZephyrSerialTransport(options) diff --git a/gallery/how_to/work_with_microtvm/micro_tvmc.sh b/gallery/how_to/work_with_microtvm/micro_tvmc.sh index 0b789216f21b..5ec718884559 100755 --- a/gallery/how_to/work_with_microtvm/micro_tvmc.sh +++ b/gallery/how_to/work_with_microtvm/micro_tvmc.sh @@ -121,7 +121,9 @@ tvmc compile magic_wand.tflite \ # # To generate a Zephyr project we use TVM Micro subcommand ``create``. We pass the MLF format and the path # for the project to ``create`` subcommand along with project options. Project options for each -# platform (Zephyr/Arduino) are defined in their Project API server file. To generate Zephyr project, run: +# platform (Zephyr/Arduino) are defined in their Project API server file. To build +# Zephyr project for a different Zephyr board, change ``zephyr_board`` project option. +# To generate Zephyr project, run: # # bash tvmc micro create \ @@ -151,11 +153,9 @@ tvmc micro create \ # bash tvmc micro build \ project \ - zephyr \ - --project-option zephyr_board=qemu_x86 + zephyr # bash -# This will build the project in ``project`` directory and generates binary files under ``project/build``. To build -# Zephyr project for a different Zephyr board, change ``zephyr_board`` project option. +# This will build the project in ``project`` directory and generates binary files under ``project/build``. # # Next, we flash the Zephyr binary file to Zephyr device. For ``qemu_x86`` Zephyr board this step does not # actually perform any action since QEMU will be used, however you need this step for physical hardware. @@ -163,8 +163,7 @@ tvmc micro build \ # bash tvmc micro flash \ project \ - zephyr \ - --project-option zephyr_board=qemu_x86 + zephyr # bash ############################################################ @@ -181,7 +180,6 @@ tvmc micro flash \ tvmc run \ --device micro \ project \ - --project-option zephyr_board=qemu_x86 \ --fill-mode ones \ --print-top 4 # bash diff --git a/tests/micro/common/test_tvmc.py b/tests/micro/common/test_tvmc.py index 096e12393d43..bd11b579e654 100644 --- a/tests/micro/common/test_tvmc.py +++ b/tests/micro/common/test_tvmc.py @@ -108,9 +108,10 @@ def test_tvmc_model_build_only(platform, board, output_dir): cmd_result = _run_tvmc(create_project_cmd) assert cmd_result == 0, "tvmc micro failed in step: create-project" - cmd_result = _run_tvmc( - ["micro", "build", project_dir, platform, "--project-option", f"{platform}_board={board}"] - ) + build_cmd = ["micro", "build", project_dir, platform] + if platform == "arduino": + build_cmd += ["--project-option", f"{platform}_board={board}"] + cmd_result = _run_tvmc(build_cmd) assert cmd_result == 0, "tvmc micro failed in step: build" shutil.rmtree(output_dir) @@ -174,28 +175,29 @@ def test_tvmc_model_run(platform, board, output_dir): cmd_result = _run_tvmc(create_project_cmd) assert cmd_result == 0, "tvmc micro failed in step: create-project" - cmd_result = _run_tvmc( - ["micro", "build", project_dir, platform, "--project-option", f"{platform}_board={board}"] - ) + build_cmd = ["micro", "build", project_dir, platform] + if platform == "arduino": + build_cmd += ["--project-option", f"{platform}_board={board}"] + cmd_result = _run_tvmc(build_cmd) + assert cmd_result == 0, "tvmc micro failed in step: build" - cmd_result = _run_tvmc( - ["micro", "flash", project_dir, platform, "--project-option", f"{platform}_board={board}"] - ) + flash_cmd = ["micro", "flash", project_dir, platform] + if platform == "arduino": + flash_cmd += ["--project-option", f"{platform}_board={board}"] + cmd_result = _run_tvmc(flash_cmd) assert cmd_result == 0, "tvmc micro failed in step: flash" - cmd_result = _run_tvmc( - [ - "run", - "--device", - "micro", - project_dir, - "--project-option", - f"{platform}_board={board}", - "--fill-mode", - "random", - ] - ) + run_cmd = [ + "run", + "--device", + "micro", + project_dir, + ] + if platform == "arduino": + run_cmd += ["--project-option", f"{platform}_board={board}"] + run_cmd += ["--fill-mode", "random"] + cmd_result = _run_tvmc(run_cmd) assert cmd_result == 0, "tvmc micro failed in step: run" shutil.rmtree(output_dir)