[Hexagon] Implement model launcher (apache#8986)

* [Hexagon] Implement model launcher

This implements a launcher that allows execution of ML models compiled
into a shared library on Hexagon DSP. It consists of two parts: the
Hexagon-side skel library and `launcher_android` to be used from
`adb shell`.

The launcher does not implement any performance-related optimizations,
it's built on top of the `graph_executor` from TVM runtime, and so it
executes a single layer at a time. This launcher should not be used to
measure performance (because if will be highly suboptimal), its main
purpose is to help in validating correctness.

* Address review comments: explanations and elaborations in README.md

* Rename cmake variables to be same as for TVM

- `HEXAGON_SDK_ROOT` -> `USE_HEXAGON_SDK`
- `HEXAGON_ARCH` -> `USE_HEXAGON_ARCH`

* Address more review comments

* Error out in cmake when USE_HEXAGON_SDK/USE_HEXAGON_ARCH are undefined

* Change FATAL_ERROR to SEND_ERROR in cmake file

cmake/modules/HexagonSDK.cmake

@@ -76,6 +76,7 @@ function(find_hexagon_sdk_root HEXAGON_SDK_PATH HEXAGON_ARCH)
   # - HEXAGON_SDK_VERSION
   # - HEXAGON_SDK_INCLUDES
   # - HEXAGON_QURT_INCLUDES
+  # - HEXAGON_QURT_LIBS
   # - HEXAGON_RPCMEM_ROOT
   # - HEXAGON_REMOTE_ROOT
   # - HEXAGON_QAIC_EXE
@@ -95,6 +96,8 @@ function(find_hexagon_sdk_root HEXAGON_SDK_PATH HEXAGON_ARCH)
     set_parent(HEXAGON_QURT_INCLUDES
       "${HEXAGON_SDK_ROOT}/libs/common/qurt/${HEXARCH_DIR}/include/posix"
       "${HEXAGON_SDK_ROOT}/libs/common/qurt/${HEXARCH_DIR}/include/qurt")
+    set_parent(HEXAGON_QURT_LIBS
+      "${HEXAGON_SDK_ROOT}/libs/common/qurt/${HEXARCH_DIR}/lib")
     set_parent(HEXAGON_RPCMEM_ROOT "${HEXAGON_SDK_ROOT}/libs/common/rpcmem")
     set_parent(HEXAGON_REMOTE_ROOT
       "${HEXAGON_SDK_ROOT}/libs/common/remote/ship/android_Release_aarch64")
@@ -111,6 +114,8 @@ function(find_hexagon_sdk_root HEXAGON_SDK_PATH HEXAGON_ARCH)
     set_parent(HEXAGON_QURT_INCLUDES
       "${HEXAGON_SDK_ROOT}/rtos/qurt/${HEXARCH_DIR}/include/posix"
       "${HEXAGON_SDK_ROOT}/rtos/qurt/${HEXARCH_DIR}/include/qurt")
+    set_parent(HEXAGON_QURT_LIBS
+      "${HEXAGON_SDK_ROOT}/rtos/qurt/${HEXARCH_DIR}/lib/pic")
     set_parent(HEXAGON_RPCMEM_ROOT "${HEXAGON_SDK_ROOT}/ipc/fastrpc/rpcmem")
     set_parent(HEXAGON_REMOTE_ROOT  # libadsprpc.so
       "${HEXAGON_SDK_ROOT}/ipc/fastrpc/remote/ship/android_aarch64")

src/runtime/hexagon/launcher/CMakeLists.txt

@@ -0,0 +1,156 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+cmake_minimum_required(VERSION 3.2)
+project(HexagonLauncher C CXX)
+
+if(NOT "${FASTRPC_LIBS}" STREQUAL "SKEL" AND
+   NOT "${FASTRPC_LIBS}" STREQUAL "STUB")
+  message(SEND_ERROR "Please set FASTRPC_LIBS to either SKEL or STUB")
+endif()
+
+if(NOT DEFINED USE_HEXAGON_SDK)
+  message(SEND_ERROR "Please set USE_HEXAGON_SDK to the location of Hexagon SDK")
+endif()
+if (NOT DEFINED USE_HEXAGON_ARCH)
+  message(SEND_ERROR "Please set USE_HEXAGON_ARCH to the Hexagon architecture version")
+endif()
+
+include(../../../../cmake/modules/HexagonSDK.cmake)
+
+find_hexagon_sdk_root("${USE_HEXAGON_SDK}" "${USE_HEXAGON_ARCH}")
+
+include_directories(SYSTEM ${HEXAGON_SDK_INCLUDES} ${HEXAGON_REMOTE_ROOT})
+
+set(QAIC_EXE "${HEXAGON_QAIC_EXE}")
+foreach(INCDIR IN LISTS HEXAGON_SDK_INCLUDES HEXAGON_REMOTE_ROOT)
+  list(APPEND QAIC_FLAGS "-I${INCDIR}")
+endforeach()
+
+set(LAUNCHER_SRC "${CMAKE_CURRENT_SOURCE_DIR}")
+set(CMAKE_SKIP_RPATH TRUE)
+
+# Qaic for the domain header.
+#
+# Don't add paths to these filenames, or otherwise cmake may spontaneously
+# add -o option to the qaic invocation (with an undesirable path).
+set(LAUNCHER_RPC_IDL "launcher_rpc.idl")
+set(LAUNCHER_RPC_H "launcher_rpc.h")
+set(LAUNCHER_RPC_SKEL_C "launcher_rpc_skel.c")
+set(LAUNCHER_RPC_STUB_C "launcher_rpc_stub.c")
+
+add_custom_command(
+  OUTPUT ${LAUNCHER_RPC_SKEL_C} ${LAUNCHER_RPC_STUB_C}
+    "${LAUNCHER_SRC}/${LAUNCHER_RPC_H}"
+  COMMAND ${QAIC_EXE} ${QAIC_FLAGS}
+    "${LAUNCHER_SRC}/${LAUNCHER_RPC_IDL}"
+  COMMAND ${CMAKE_COMMAND} -E rename "${LAUNCHER_RPC_H}"
+    "${LAUNCHER_SRC}/${LAUNCHER_RPC_H}"
+  MAIN_DEPENDENCY "${LAUNCHER_SRC}/${LAUNCHER_RPC_IDL}"
+)
+
+
+if("${FASTRPC_LIBS}" STREQUAL "SKEL")
+  # Skel libraries.
+  #
+  if (NOT DEFINED TVM_RUNTIME_HEXAGON)
+    message(SEND_ERROR "Please set TVM_RUNTIME_HEXAGON=/path/to/libtvm_runtime.a")
+  endif()
+
+  include_directories(SYSTEM ${HEXAGON_QURT_INCLUDES})
+  include_directories(
+    "${LAUNCHER_SRC}"
+    "${LAUNCHER_SRC}/../../../../include"
+    "${LAUNCHER_SRC}/../../../../3rdparty/dlpack/include"
+    "${LAUNCHER_SRC}/../../../../3rdparty/dmlc-core/include"
+  )
+  link_directories(${HEXAGON_QURT_LIBS})
+
+  add_definitions(-D_MACH_I32=int)
+  add_definitions(-DDMLC_CXX11_THREAD_LOCAL=0)
+  add_definitions(-DDMLC_USE_LOGGING_LIBRARY=<tvm/runtime/logging.h>)
+
+  # Extra compile flags (both C and C++).
+  set(EXTRA_COMP_FLAGS
+    "-O3"
+    "-m${USE_HEXAGON_ARCH}"
+  )
+  string(REGEX REPLACE ";" " " EXTRA_COMP_FLAGS_STR "${EXTRA_COMP_FLAGS}")
+  set(CMAKE_C_FLAGS "${EXTRA_COMP_FLAGS_STR} ${CMAKE_C_FLAGS}")
+  set(CMAKE_CXX_FLAGS "${EXTRA_COMP_FLAGS_STR} ${CMAKE_CXX_FLAGS}")
+
+  set(EXTRA_LINK_FLAGS
+    "-lposix"
+    "-lqurt"
+    "-Wl,--export-dynamic"
+    "-Wl,--whole-archive ${TVM_RUNTIME_HEXAGON} -Wl,--no-whole-archive"
+    "-Wl,--defsym=HEAP_SIZE=0x40000000"
+  )
+  string(REGEX REPLACE ";" " " EXTRA_LINK_FLAGS_STR "${EXTRA_LINK_FLAGS}")
+
+  set(SKEL_SRCS
+    "launcher_core.cc"
+    "launcher_hexagon.cc"
+  )
+  add_library(launcher_rpc_skel SHARED
+    "${LAUNCHER_SRC}/${LAUNCHER_RPC_H}"
+    "${LAUNCHER_RPC_SKEL_C}"
+    "${SKEL_SRCS}"
+  )
+
+  # Extra linker flags for linking shared libraries.
+  set_target_properties(launcher_rpc_skel PROPERTIES
+    LINK_FLAGS ${EXTRA_LINK_FLAGS_STR}
+  )
+else()
+  # Stub libraries.
+  #
+  if (NOT DEFINED TVM_RUNTIME_ANDROID)
+    message(SEND_ERROR "Please set TVM_RUNTIME_ANDROID=/path/to/libtvm_runtime.so")
+  endif()
+
+  include_directories(SYSTEM
+    "${HEXAGON_SDK_INCLUDES}"
+    "${HEXAGON_RPCMEM_ROOT}/inc"
+  )
+  include_directories(
+    "${LAUNCHER_SRC}"
+    "${LAUNCHER_SRC}/../../../../include"
+    "${LAUNCHER_SRC}/../../../../3rdparty/dlpack/include"
+    "${LAUNCHER_SRC}/../../../../3rdparty/dmlc-core/include"
+  )
+  link_directories(${HEXAGON_REMOTE_ROOT})
+
+  add_definitions(-DDMLC_USE_LOGGING_LIBRARY=<tvm/runtime/logging.h>)
+
+  set(STUB_SRCS
+    "launcher_android.cc"
+    "launcher_core.cc"
+    "launcher_main.cc"
+    "launcher_util.cc"
+  )
+
+  add_executable(launcher_android
+    "${STUB_SRCS}"
+    "${LAUNCHER_RPC_STUB_C}"
+  )
+  target_link_libraries(launcher_android cdsprpc log)
+
+  set_target_properties(launcher_android PROPERTIES
+    LINK_FLAGS "${TVM_RUNTIME_ANDROID}"
+  )
+endif()

src/runtime/hexagon/launcher/README.md

@@ -0,0 +1,175 @@
+<!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+# Hexagon Graph Launcher
+
+## Compilation
+
+The launcher consists of two parts: part running on Hexagon, and part running
+on Android. They need to be compiled separately. Since some source files are
+shared between these two parts, make sure to delete all object files between
+compilations. Compile the Hexagon code first.
+
+The supported Snapdragon architectures are 855, 865, and 888.
+
+### Prerequisites
+
+1. Android NDK version r19c or later.
+2. Hexagon SDK version 4.0.0 or later.
+
+Android NDK can be downloaded from https://developer.android.com/ndk.
+Hexagon SDK is available at //developer.qualcomm.com/software/hexagon-dsp-sdk.
+
+### Compilation of the Hexagon part
+
+1. Build the static version of TVM runtime for Hexagon. Use Hexagon clang
+   from the Hexagon SDK. This step is the same as building the shared version,
+   except at the cmake step, add `-DBUILD_STATIC_RUNTIME=ON`. The compilation
+   step should create `libtvm_runtime.a`.
+
+2. Create a subdirectory for the build files, and run `cmake` with the
+   following variables set:
+   - `FASTRPC_LIBS=SKEL`
+   - `USE_HEXAGON_SDK` to the path to the Hexagon SDK
+   - `CMAKE_C_COMPILER=hexagon-clang`
+   - `CMAKE_CXX_COMPILER=hexagon-clang++`
+   - `USE_HEXAGON_ARCH` to one of v65, v66, v68
+   - `TVM_RUNTIME_HEXAGON=/path/to/libtvm_runtime.a` _statically_ linked
+     TVM runtime
+
+   Make sure to provide the path to launcher's `CMakeLists.txt` directory
+   in `cmake` invocation.
+
+3. Run `make`. This will create `liblauncher_rpc_skel.so`.
+
+### Compilation of the Android part
+
+1. Build TVM runtime for Android, using clang for AArch64 from the Android
+   NDK. Unlike in the Hexagon case, this should be the dynamic library (which
+   is the default), i.e. `libtvm_runtime.so`.
+
+2. Create a subdirectory for the build files (different from the one used for
+   Hexagon files), and run `cmake` with the following variables set:
+   - `FASTRPC_LIBS=STUB`
+   - `USE_HEXAGON_SDK` to the path to the Hexagon SDK
+   - `CMAKE_C_COMPILER=aarch64-linux-android28-clang` (or later)
+   - `CMAKE_CXX_COMPILER=aarch64-linux-android28-clang++` (or later)
+   - `USE_HEXAGON_ARCH` to one of v65, v66, v68 (same as for the Hexagon part)
+   - `TVM_RUNTIME_ANDROID=/path/to/libtvm_runtime.so` dynamically or
+     statically linked TVM runtime
+
+3. Run `make`. This will create `launcher_android`.
+
+## Execution
+
+From the Android shell, do
+```
+./launcher_android --in_config input.json --out_config output.json
+```
+
+You may need to add the location of `libtvm_runtime.so` to `LD_LIBRARY_PATH`.
+See below for more information about the setup and launcher's inputs.
+
+### Preparation steps
+
+Copy the following binaries to the device:
+- `liblauncher_rpc_skel.so`: created by the compilation step for Hexagon,
+- `libgcc.so`: take this one from the Hexagon toolchain,
+- `launcher_android`: created by the compilation step for Android,
+- `libtvm_runtime.so`: built for Android.
+
+These are only the binaries related to the launcher itself. To run a model
+copy the shared object with the model and the model JSON file over to the
+device (both are obtained from relay).  Also, copy all input files for the
+model as well.
+
+The following snippet illustrates how to obtain the shared object and the
+JSON file from a TFLite model (using Inception V3 as an example):
+
+```
+# Skipped imports, etc.
+
+with open("inception_v3.tflite", "rb") as f:
+    tflite_model_buf = f.read()
+tflite_model = tflite.Model.Model.GetRootAsModel(tflite_model_buf, 0)
+
+shape_dict = { "input": [1,299,299,3] }
+dtype_dict = { "input": "float32" }
+
+mod, params = relay.frontend.from_tflite(
+    tflite_model, shape_dict=shape_dict, dtype_dict=dtype_dict
+)
+
+target = tvm.target.hexagon('v68', link_params=True)
+with tvm.transform.PassContext(opt_level=3):
+    lib = relay.build(mod, target, target_host=target, params=params, mod_name="default")
+
+# Save model.so and model.json:
+with open('model.json', 'w') as f:
+    f.write(lib.get_graph_json())
+lib.get_lib().save('model.so')
+```
+
+The final thing is to prepare a JSON configuration file for the launcher.
+The JSON has two attributes describing the model: `model-library` and
+`model-json`, and an attribute `inputs`, which is a list of records, one
+for each input file.
+An input file record has three attributes: `file`, `shape`, and `dtype`.
+
+Below is an example of the input config file for Inception V3:
+```
+{
+  "model-library": "inceptionv3-float32.so",
+  "model-json": "inceptionv3-float32.json",
+  "inputs" : [
+    {
+      "file": "panda_299x299_fp.dat",
+      "shape": [1,299,299,3],
+      "dtype": "float32"
+    }
+  ]
+}
+```
+
+The launcher will then create the output JSON file (with the name given via
+`--out_config`) containing information about the execution time and the model
+outputs. The output JSON file has three attributes: "pcycles", "usecs" that
+contain the execution duration in terms of processor cycles and microseconds
+respectivaly, and an attribute `outputs`, which is a list of output file records
+whose syntax is identical to the input file records in the input file.
+A sample output JSON from running the Inception V3 model may look like
+```
+{
+  "pcycles": 112965680178,
+  "usecs": 79532302,
+  "outputs": [
+    {
+      "file": "output0.dat",
+      "shape": [1, 1001],
+      "dtype": "float32"
+    }
+  ]
+}
+```
+
+# Disclaimer
+
+The launcher does not perform any correctness verification. In order to verify
+correctness, the user needs to copy the output files from the device and
+verify their contents.
+
+This launcher is intended for use with prototyping and does not utilize any
+performance acceleration, as such the measured performance may be very poor.