[microTVM] Autotuning performance tests (apache#11782)

* Common autotuning test

* Autotuned model evaluation utilities

* Bugfixes and more enablement

* Working autotune profiling test

* Refactoring based on PR comments

Bugfixes to get tests passing

Refactor to remove tflite model for consistency

Black formatting

Linting and bugfixes

Add Apache license header

Use larger chunk size to read files

Explicitly specify LRU cache size for compatibility with Python 3.7

Pass platform to microTVM common tests

Better comment for runtime bound

Stop directory from being removed after session creation

* Use the actual Zephyr timing library

Use unsigned integer

Additional logging

Try negation

Try 64 bit timer

Use Zephyr's timing library

Fix linting

Enable timing utilities

apps/microtvm/zephyr/template_project/microtvm_api_server.py

@@ -393,6 +393,7 @@ def _create_prj_conf(self, project_dir, options):
 
             if options["project_type"] == "host_driven":
                 f.write(
+                    "CONFIG_TIMING_FUNCTIONS=y\n"
                     "# For RPC server C++ bindings.\n"
                     "CONFIG_CPLUSPLUS=y\n"
                     "CONFIG_LIB_CPLUSPLUS=y\n"

apps/microtvm/zephyr/template_project/src/host_driven/main.c

@@ -38,6 +38,7 @@
 #include <sys/printk.h>
 #include <sys/reboot.h>
 #include <sys/ring_buffer.h>
+#include <timing/timing.h>
 #include <tvm/runtime/crt/logging.h>
 #include <tvm/runtime/crt/microtvm_rpc_server.h>
 #include <unistd.h>
@@ -144,11 +145,7 @@ tvm_crt_error_t TVMPlatformMemoryFree(void* ptr, DLDevice dev) {
   return kTvmErrorNoError;
 }
 
-#define MILLIS_TIL_EXPIRY 200
-#define TIME_TIL_EXPIRY (K_MSEC(MILLIS_TIL_EXPIRY))
-K_TIMER_DEFINE(g_microtvm_timer, /* expiry func */ NULL, /* stop func */ NULL);
-
-uint32_t g_microtvm_start_time;
+volatile timing_t g_microtvm_start_time, g_microtvm_end_time;
 int g_microtvm_timer_running = 0;
 
 // Called to start system timer.
@@ -161,8 +158,7 @@ tvm_crt_error_t TVMPlatformTimerStart() {
 #ifdef CONFIG_LED
   gpio_pin_set(led0_pin, LED0_PIN, 1);
 #endif
-  k_timer_start(&g_microtvm_timer, TIME_TIL_EXPIRY, TIME_TIL_EXPIRY);
-  g_microtvm_start_time = k_cycle_get_32();
+  g_microtvm_start_time = timing_counter_get();
   g_microtvm_timer_running = 1;
   return kTvmErrorNoError;
 }
@@ -174,43 +170,14 @@ tvm_crt_error_t TVMPlatformTimerStop(double* elapsed_time_seconds) {
     return kTvmErrorSystemErrorMask | 2;
   }
 
-  uint32_t stop_time = k_cycle_get_32();
 #ifdef CONFIG_LED
   gpio_pin_set(led0_pin, LED0_PIN, 0);
 #endif
 
-  // compute how long the work took
-  uint32_t cycles_spent = stop_time - g_microtvm_start_time;
-  if (stop_time < g_microtvm_start_time) {
-    // we rolled over *at least* once, so correct the rollover it was *only*
-    // once, because we might still use this result
-    cycles_spent = ~((uint32_t)0) - (g_microtvm_start_time - stop_time);
-  }
-
-  uint32_t ns_spent = (uint32_t)k_cyc_to_ns_floor64(cycles_spent);
-  double hw_clock_res_us = ns_spent / 1000.0;
-
-  // need to grab time remaining *before* stopping. when stopped, this function
-  // always returns 0.
-  int32_t time_remaining_ms = k_timer_remaining_get(&g_microtvm_timer);
-  k_timer_stop(&g_microtvm_timer);
-  // check *after* stopping to prevent extra expiries on the happy path
-  if (time_remaining_ms < 0) {
-    TVMLogf("negative time remaining");
-    return kTvmErrorSystemErrorMask | 3;
-  }
-  uint32_t num_expiries = k_timer_status_get(&g_microtvm_timer);
-  uint32_t timer_res_ms = ((num_expiries * MILLIS_TIL_EXPIRY) + time_remaining_ms);
-  double approx_num_cycles =
-      (double)k_ticks_to_cyc_floor32(1) * (double)k_ms_to_ticks_ceil32(timer_res_ms);
-  // if we approach the limits of the HW clock datatype (uint32_t), use the
-  // coarse-grained timer result instead
-  if (approx_num_cycles > (0.5 * (~((uint32_t)0)))) {
-    *elapsed_time_seconds = timer_res_ms / 1000.0;
-  } else {
-    *elapsed_time_seconds = hw_clock_res_us / 1e6;
-  }
-
+  g_microtvm_end_time = timing_counter_get();
+  uint64_t cycles = timing_cycles_get(&g_microtvm_start_time, &g_microtvm_end_time);
+  uint64_t ns_spent = timing_cycles_to_ns(cycles);
+  *elapsed_time_seconds = ns_spent / (double)1e9;
   g_microtvm_timer_running = 0;
   return kTvmErrorNoError;
 }
@@ -278,6 +245,11 @@ void main(void) {
   tvm_uart = device_get_binding(DT_LABEL(DT_CHOSEN(zephyr_console)));
   uart_rx_init(&uart_rx_rbuf, tvm_uart);
 
+  // Initialize system timing. We could stop and start it every time, but we'll
+  // be using it enough we should just keep it enabled.
+  timing_init();
+  timing_start();
+
   // Initialize microTVM RPC server, which will receive commands from the UART and execute them.
   microtvm_rpc_server_t server = MicroTVMRpcServerInit(write_serial, NULL);
   TVMLogf("microTVM Zephyr runtime - running");

python/tvm/micro/testing/__init__.py

@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""Allows the tools specified below to be imported directly from tvm.micro.testing"""
+from .evaluation import tune_model, create_aot_session, evaluate_model_accuracy
+from .utils import get_supported_boards, get_target

python/tvm/micro/testing/aot_test_utils.py

@@ -15,17 +15,22 @@
 # specific language governing permissions and limitations
 # under the License.
 
+"""
+This file provides utilities for running AOT tests, especially for Corstone.
+
+"""
+
 import logging
 import itertools
 import shutil
 
 import pytest
 
-pytest.importorskip("tvm.micro")
-
 import tvm
 from tvm.testing.aot import AOTTestRunner
 
+pytest.importorskip("tvm.micro")
+
 _LOG = logging.getLogger(__name__)
 
 
@@ -97,9 +102,9 @@ def parametrize_aot_options(test):
         valid_combinations,
     )
 
-    fn = pytest.mark.parametrize(
+    func = pytest.mark.parametrize(
         ["interface_api", "use_unpacked_api", "test_runner"],
         marked_combinations,
     )(test)
 
-    return tvm.testing.skip_if_32bit(reason="Reference system unavailable in i386 container")(fn)
+    return tvm.testing.skip_if_32bit(reason="Reference system unavailable in i386 container")(func)

python/tvm/micro/testing/evaluation.py

@@ -0,0 +1,150 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Provides high-level functions for instantiating and timing AOT models. Used
+by autotuning tests in tests/micro, and may be used for more performance
+tests in the future.
+
+"""
+
+from io import StringIO
+from pathlib import Path
+from contextlib import ExitStack
+import tempfile
+
+import tvm
+
+
+def tune_model(
+    platform, board, target, mod, params, num_trials, tuner_cls=tvm.autotvm.tuner.GATuner
+):
+    """Autotunes a model with microTVM and returns a StringIO with the tuning logs"""
+    with tvm.transform.PassContext(opt_level=3, config={"tir.disable_vectorize": True}):
+        tasks = tvm.autotvm.task.extract_from_program(mod["main"], {}, target)
+    assert len(tasks) > 0
+    assert isinstance(params, dict)
+
+    module_loader = tvm.micro.AutoTvmModuleLoader(
+        template_project_dir=tvm.micro.get_microtvm_template_projects(platform),
+        project_options={
+            f"{platform}_board": board,
+            "project_type": "host_driven",
+        },
+    )
+
+    builder = tvm.autotvm.LocalBuilder(
+        n_parallel=1,
+        build_kwargs={"build_option": {"tir.disable_vectorize": True}},
+        do_fork=False,
+        build_func=tvm.micro.autotvm_build_func,
+        runtime=tvm.relay.backend.Runtime("crt", {"system-lib": True}),
+    )
+    runner = tvm.autotvm.LocalRunner(number=1, repeat=1, timeout=100, module_loader=module_loader)
+    measure_option = tvm.autotvm.measure_option(builder=builder, runner=runner)
+
+    results = StringIO()
+    for task in tasks:
+        tuner = tuner_cls(task)
+
+        tuner.tune(
+            n_trial=num_trials,
+            measure_option=measure_option,
+            callbacks=[
+                tvm.autotvm.callback.log_to_file(results),
+                tvm.autotvm.callback.progress_bar(num_trials, si_prefix="M"),
+            ],
+            si_prefix="M",
+        )
+        assert tuner.best_flops > 1
+
+    return results
+
+
+def create_aot_session(
+    platform,
+    board,
+    target,
+    mod,
+    params,
+    build_dir=Path(tempfile.mkdtemp()),
+    tune_logs=None,
+    use_cmsis_nn=False,
+):
+    """AOT-compiles and uploads a model to a microcontroller, and returns the RPC session"""
+
+    executor = tvm.relay.backend.Executor("aot")
+    crt_runtime = tvm.relay.backend.Runtime("crt", {"system-lib": True})
+
+    with ExitStack() as stack:
+        config = {"tir.disable_vectorize": True}
+        if use_cmsis_nn:
+            config["relay.ext.cmsisnn.options"] = {"mcpu": target.mcpu}
+        stack.enter_context(tvm.transform.PassContext(opt_level=3, config=config))
+        if tune_logs is not None:
+            stack.enter_context(tvm.autotvm.apply_history_best(tune_logs))
+
+        lowered = tvm.relay.build(
+            mod,
+            target=target,
+            params=params,
+            runtime=crt_runtime,
+            executor=executor,
+        )
+    parameter_size = len(tvm.runtime.save_param_dict(lowered.get_params()))
+    print(f"Model parameter size: {parameter_size}")
+
+    # Once the project has been uploaded, we don't need to keep it
+    project = tvm.micro.generate_project(
+        str(tvm.micro.get_microtvm_template_projects(platform)),
+        lowered,
+        build_dir / "project",
+        {
+            f"{platform}_board": board,
+            "project_type": "host_driven",
+        },
+    )
+    project.build()
+    project.flash()
+
+    return tvm.micro.Session(project.transport())
+
+
+# This utility functions was designed ONLY for one input / one output models
+# where the outputs are confidences for different classes.
+def evaluate_model_accuracy(session, aot_executor, input_data, true_labels, runs_per_sample=1):
+    """Evaluates an AOT-compiled model's accuracy and runtime over an RPC session. Works well
+    when used with create_aot_session."""
+
+    assert aot_executor.get_num_inputs() == 1
+    assert aot_executor.get_num_outputs() == 1
+    assert runs_per_sample > 0
+
+    predicted_labels = []
+    aot_runtimes = []
+    for sample in input_data:
+        aot_executor.get_input(0).copyfrom(sample)
+        result = aot_executor.module.time_evaluator("run", session.device, number=runs_per_sample)()
+        runtime = result.mean
+        output = aot_executor.get_output(0).numpy()
+        predicted_labels.append(output.argmax())
+        aot_runtimes.append(runtime)
+
+    num_correct = sum(u == v for u, v in zip(true_labels, predicted_labels))
+    average_time = sum(aot_runtimes) / len(aot_runtimes)
+    accuracy = num_correct / len(predicted_labels)
+    return average_time, accuracy

python/tvm/micro/testing/utils.py

@@ -17,9 +17,10 @@
 
 """Defines the test methods used with microTVM."""
 
-import pathlib
+from functools import lru_cache
 import json
 import logging
+from pathlib import Path
 import tarfile
 import time
 from typing import Union
@@ -32,7 +33,19 @@
 TIMEOUT_SEC = 10
 
 
-def check_tune_log(log_path: Union[pathlib.Path, str]):
+@lru_cache(maxsize=None)
+def get_supported_boards(platform: str):
+    template = Path(tvm.micro.get_microtvm_template_projects(platform))
+    with open(template / "boards.json") as f:
+        return json.load(f)
+
+
+def get_target(platform: str, board: str):
+    model = get_supported_boards(platform)[board]["model"]
+    return str(tvm.target.target.micro(model))
+
+
+def check_tune_log(log_path: Union[Path, str]):
     """Read the tuning log and check each result."""
     with open(log_path, "r") as f:
         lines = f.readlines()
@@ -76,7 +89,7 @@ def _read_line(transport, timeout_sec: int) -> str:
                 return data.decode(encoding="utf-8")
 
 
-def mlf_extract_workspace_size_bytes(mlf_tar_path: Union[pathlib.Path, str]) -> int:
+def mlf_extract_workspace_size_bytes(mlf_tar_path: Union[Path, str]) -> int:
     """Extract an MLF archive file and read workspace size from metadata file."""
 
     workspace_size = 0