-
Notifications
You must be signed in to change notification settings - Fork 3.4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[Hexagon] Implement model launcher #8986
Changes from 1 commit
7f736df
505dbdc
6e7a398
1f2ea92
72cf0c0
4bb5e9f
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,149 @@ | ||
# Licensed to the Apache Software Foundation (ASF) under one | ||
# or more contributor license agreements. See the NOTICE file | ||
# distributed with this work for additional information | ||
# regarding copyright ownership. The ASF licenses this file | ||
# to you under the Apache License, Version 2.0 (the | ||
# "License"); you may not use this file except in compliance | ||
# with the License. You may obtain a copy of the License at | ||
# | ||
# http://www.apache.org/licenses/LICENSE-2.0 | ||
# | ||
# Unless required by applicable law or agreed to in writing, | ||
# software distributed under the License is distributed on an | ||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
# KIND, either express or implied. See the License for the | ||
# specific language governing permissions and limitations | ||
# under the License. | ||
|
||
cmake_minimum_required(VERSION 3.2) | ||
project(HexagonLauncher C CXX) | ||
|
||
if(NOT "${FASTRPC_LIBS}" STREQUAL "SKEL" AND | ||
NOT "${FASTRPC_LIBS}" STREQUAL "STUB") | ||
message(SEND_ERROR "Please set FASTRPC_LIBS to either SKEL or STUB") | ||
endif() | ||
|
||
include(../../../../cmake/modules/HexagonSDK.cmake) | ||
|
||
find_hexagon_sdk_root("${HEXAGON_SDK_ROOT}" "${HEXAGON_ARCH}") | ||
|
||
include_directories(SYSTEM ${HEXAGON_SDK_INCLUDES} ${HEXAGON_REMOTE_ROOT}) | ||
|
||
set(QAIC_EXE "${HEXAGON_QAIC_EXE}") | ||
foreach(INCDIR IN LISTS HEXAGON_SDK_INCLUDES HEXAGON_REMOTE_ROOT) | ||
list(APPEND QAIC_FLAGS "-I${INCDIR}") | ||
endforeach() | ||
|
||
set(LAUNCHER_SRC "${CMAKE_CURRENT_SOURCE_DIR}") | ||
set(CMAKE_SKIP_RPATH TRUE) | ||
|
||
# Qaic for the domain header. | ||
# | ||
# Don't add paths to these filenames, or otherwise cmake may spontaneously | ||
# add -o option to the qaic invocation (with an undesirable path). | ||
set(LAUNCHER_RPC_IDL "launcher_rpc.idl") | ||
set(LAUNCHER_RPC_H "launcher_rpc.h") | ||
set(LAUNCHER_RPC_SKEL_C "launcher_rpc_skel.c") | ||
set(LAUNCHER_RPC_STUB_C "launcher_rpc_stub.c") | ||
|
||
add_custom_command( | ||
OUTPUT ${LAUNCHER_RPC_SKEL_C} ${LAUNCHER_RPC_STUB_C} | ||
"${LAUNCHER_SRC}/${LAUNCHER_RPC_H}" | ||
COMMAND ${QAIC_EXE} ${QAIC_FLAGS} | ||
"${LAUNCHER_SRC}/${LAUNCHER_RPC_IDL}" | ||
COMMAND ${CMAKE_COMMAND} -E rename "${LAUNCHER_RPC_H}" | ||
"${LAUNCHER_SRC}/${LAUNCHER_RPC_H}" | ||
MAIN_DEPENDENCY "${LAUNCHER_SRC}/${LAUNCHER_RPC_IDL}" | ||
) | ||
|
||
|
||
if("${FASTRPC_LIBS}" STREQUAL "SKEL") | ||
# Skel libraries. | ||
# | ||
if (NOT DEFINED TVM_RUNTIME_HEXAGON) | ||
message(FATAL_ERROR "Please set TVM_RUNTIME_HEXAGON=/path/to/libtvm_runtime.a") | ||
endif() | ||
|
||
include_directories(SYSTEM ${HEXAGON_QURT_INCLUDES}) | ||
include_directories( | ||
"${LAUNCHER_SRC}" | ||
"${LAUNCHER_SRC}/../../../../include" | ||
"${LAUNCHER_SRC}/../../../../3rdparty/dlpack/include" | ||
"${LAUNCHER_SRC}/../../../../3rdparty/dmlc-core/include" | ||
) | ||
link_directories(${HEXAGON_QURT_LIBS}) | ||
|
||
add_definitions(-D_MACH_I32=int) | ||
add_definitions(-DDMLC_CXX11_THREAD_LOCAL=0) | ||
add_definitions(-DDMLC_USE_LOGGING_LIBRARY=<tvm/runtime/logging.h>) | ||
|
||
# Extra compile flags (both C and C++). | ||
set(EXTRA_COMP_FLAGS | ||
"-O3" | ||
"-m${HEXAGON_ARCH}" | ||
) | ||
string(REGEX REPLACE ";" " " EXTRA_COMP_FLAGS_STR "${EXTRA_COMP_FLAGS}") | ||
set(CMAKE_C_FLAGS "${EXTRA_COMP_FLAGS_STR} ${CMAKE_C_FLAGS}") | ||
set(CMAKE_CXX_FLAGS "${EXTRA_COMP_FLAGS_STR} ${CMAKE_CXX_FLAGS}") | ||
|
||
set(EXTRA_LINK_FLAGS | ||
"-lposix" | ||
"-lqurt" | ||
"-Wl,--export-dynamic" | ||
"-Wl,--whole-archive ${TVM_RUNTIME_HEXAGON} -Wl,--no-whole-archive" | ||
"-Wl,--defsym=HEAP_SIZE=0x40000000" | ||
) | ||
string(REGEX REPLACE ";" " " EXTRA_LINK_FLAGS_STR "${EXTRA_LINK_FLAGS}") | ||
|
||
set(SKEL_SRCS | ||
"launcher_core.cc" | ||
"launcher_hexagon.cc" | ||
) | ||
add_library(launcher_rpc_skel SHARED | ||
"${LAUNCHER_SRC}/${LAUNCHER_RPC_H}" | ||
"${LAUNCHER_RPC_SKEL_C}" | ||
"${SKEL_SRCS}" | ||
) | ||
|
||
# Extra linker flags for linking shared libraries. | ||
set_target_properties(launcher_rpc_skel PROPERTIES | ||
LINK_FLAGS ${EXTRA_LINK_FLAGS_STR} | ||
) | ||
else() | ||
# Stub libraries. | ||
# | ||
if (NOT DEFINED TVM_RUNTIME_ANDROID) | ||
message(FATAL_ERROR "Please set TVM_RUNTIME_ANDROID=/path/to/libtvm_runtime.so") | ||
endif() | ||
|
||
include_directories(SYSTEM | ||
"${HEXAGON_SDK_INCLUDES}" | ||
"${HEXAGON_RPCMEM_ROOT}/inc" | ||
) | ||
include_directories( | ||
"${LAUNCHER_SRC}" | ||
"${LAUNCHER_SRC}/../../../../include" | ||
"${LAUNCHER_SRC}/../../../../3rdparty/dlpack/include" | ||
"${LAUNCHER_SRC}/../../../../3rdparty/dmlc-core/include" | ||
) | ||
link_directories(${HEXAGON_REMOTE_ROOT}) | ||
|
||
add_definitions(-DDMLC_USE_LOGGING_LIBRARY=<tvm/runtime/logging.h>) | ||
|
||
set(STUB_SRCS | ||
"launcher_android.cc" | ||
"launcher_core.cc" | ||
"launcher_main.cc" | ||
"launcher_util.cc" | ||
) | ||
|
||
add_executable(launcher_android | ||
"${STUB_SRCS}" | ||
"${LAUNCHER_RPC_STUB_C}" | ||
) | ||
target_link_libraries(launcher_android cdsprpc log) | ||
|
||
set_target_properties(launcher_android PROPERTIES | ||
LINK_FLAGS "${TVM_RUNTIME_ANDROID}" | ||
) | ||
endif() |
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
@@ -0,0 +1,142 @@ | ||||||
<!--- Licensed to the Apache Software Foundation (ASF) under one --> | ||||||
<!--- or more contributor license agreements. See the NOTICE file --> | ||||||
<!--- distributed with this work for additional information --> | ||||||
<!--- regarding copyright ownership. The ASF licenses this file --> | ||||||
<!--- to you under the Apache License, Version 2.0 (the --> | ||||||
<!--- "License"); you may not use this file except in compliance --> | ||||||
<!--- with the License. You may obtain a copy of the License at --> | ||||||
|
||||||
<!--- http://www.apache.org/licenses/LICENSE-2.0 --> | ||||||
|
||||||
<!--- Unless required by applicable law or agreed to in writing, --> | ||||||
<!--- software distributed under the License is distributed on an --> | ||||||
<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY --> | ||||||
<!--- KIND, either express or implied. See the License for the --> | ||||||
<!--- specific language governing permissions and limitations --> | ||||||
<!--- under the License. --> | ||||||
# Hexagon Graph Launcher | ||||||
|
||||||
## Compilation | ||||||
|
||||||
The launcher consists of two parts: part running on Hexagon, and part running | ||||||
on Android. They need to be compiled separately. Since some source files are | ||||||
shared between these two parts, make sure to delete all object files beteween | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: between There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed. |
||||||
compilations. Compile the Hexagon code first. | ||||||
|
||||||
### Prerequisites | ||||||
|
||||||
1. Android NDK version r19c or later. | ||||||
2. Hexagon SDK version 4.0.0 or later. | ||||||
|
||||||
Android NDK can be downloaded from https://developer.android.com/ndk. | ||||||
Hexagon SDK is available at //developer.qualcomm.com/software/hexagon-dsp-sdk. | ||||||
|
||||||
### Compilation of the Hexagon part | ||||||
|
||||||
1. Build the static version of TVM runtime for Hexagon: this step is the same | ||||||
as building the shared version, except at the cmake step, add | ||||||
`-DBUILD_STATIC_RUNTIME=ON`. The compilation step should create | ||||||
`libtvm_runtime.a`. | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Elaborate here to include that the the compiler used here should be the hexagon-clang compiler from the hexagon toolchain. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done. |
||||||
|
||||||
2. Create a subdirectory for the build files, and run `cmake` with the | ||||||
following variables set: | ||||||
- `FASTRPC_LIBS=SKEL` | ||||||
- `HEXAGON_SDK_ROOT` to the path to the Hexagon SDK | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
nit: would be nice to normalize to the naming convention used for the hexagon cmake variables in TVM. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done. |
||||||
- `CMAKE_C_COMPILER=hexagon-clang` | ||||||
- `CMAKE_CXX_COMPILER=hexagon-clang++` | ||||||
- `HEXAGON_ARCH` to one of v65, v66, v68 | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
nit: would be nice to normalize to the naming convention used for the hexagon cmake variables in TVM. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done. |
||||||
- `TVM_RUNTIME_HEXAGON=/path/to/libtvm_runtime.a` _statically_ linked | ||||||
TVM runtime | ||||||
Make sure to provide the path to launcher's `CMakeLists.txt` directory | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You need an extra space here otherwise this line appears as a continuation of the previous bullet. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed. |
||||||
in `cmake` invocation. | ||||||
|
||||||
3. Run `make`. This will create `liblauncher_rpc_skel.so`. | ||||||
|
||||||
### Compilation of the Android part | ||||||
|
||||||
1. Build TVM runtime for Android. Unlike in the Hexagon case, this should be | ||||||
the dynamic library (which is the default), i.e. `libtvm_runtime.so`. | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Elaborate here to include that the the compiler used here should be the aarch64 linux compiler for android. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done. |
||||||
|
||||||
2. Create a subdirectory for the build files (different from the one used for | ||||||
Hexagon files), and run `cmake` with the following variables set: | ||||||
- `FASTRPC_LIBS=STUB` | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. When I follow these instructions and try to run the launcher on an 888 device I am seeing an error opening the FastRPC channel. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. There can be a number of reasons for that. The diagnostic output from There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is it worth adding a comment about using |
||||||
- `HEXAGON_SDK_ROOT` to the path to the Hexagon SDK | ||||||
- `CMAKE_C_COMPILER=aarch64-linux-android28-clang` (or later) | ||||||
- `CMAKE_CXX_COMPILER=aarch64-linux-android28-clang++` (or later) | ||||||
- `HEXAGON_ARCH` to one of v65, v66, v68 (same as for the Hexagon part) | ||||||
- `TVM_RUNTIME_ANDROID=/path/to/libtvm_runtime.so` dynamically or | ||||||
statically linked TVM runtime | ||||||
|
||||||
3. Run `make`. This will create `launcher_android`. | ||||||
|
||||||
## Execution | ||||||
|
||||||
From the Android shell, do | ||||||
``` | ||||||
./launcher_android --in_config input.json --out_config output.json | ||||||
``` | ||||||
|
||||||
You may need to add the location of `libtvm_runtime.so` to `LD_LIBRARY_PATH`. | ||||||
See below for more information about the setup and launcher's inputs. | ||||||
|
||||||
### Preparation steps | ||||||
|
||||||
Copy the set of binaries created in the compilation step to the device: | ||||||
- `liblauncher_rpc_skel.so`, | ||||||
- `libgcc.so` (this one should come from the Hexagon toolchain), | ||||||
- `launcher_android`, | ||||||
- `libtvm_runtime.so` (for Android). | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. maybe clarify: for the Android-side binary or even There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think I addressed it, let me know if that's what you meant. |
||||||
|
||||||
These are only the binaries related to the launcher itself. To run a model | ||||||
copy the shared object with the model and the model JSON file over to the | ||||||
device (both are obtained from relay). Also, copy all input files for the | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It would be helpful for the general public to indicate how these can be produced out of relay with a small python snippet. Or have a small python script that does this in the same way of the howtodeploy example: https://github.com/apache/tvm/blob/main/apps/howto_deploy/prepare_test_libs.py There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done. |
||||||
model as well. | ||||||
|
||||||
The final thing is to prepare a JSON configuration file for the launcher. | ||||||
The JSON has two attributes describing the model: `model-library` and | ||||||
`model-json`, and an attribute `inputs`, which is a list of records, one | ||||||
for each input file. | ||||||
An input file record has three attributes: `file`, `shape`, and `dtype`. | ||||||
|
||||||
Below is an example of the input config file for Inception V3: | ||||||
``` | ||||||
{ | ||||||
"model-library": "inceptionv3-float32.so", | ||||||
"model-json": "inceptionv3-float32.json", | ||||||
"inputs" : [ | ||||||
{ | ||||||
"file": "panda_299x299_fp.dat", | ||||||
"shape": [1,299,299,3], | ||||||
"dtype": "float32" | ||||||
} | ||||||
] | ||||||
} | ||||||
``` | ||||||
|
||||||
The launcher will then create the output JSON file (with the name given via | ||||||
`--out_config`) containing information about the execution time and the model | ||||||
outputs. The output JSON file has three attributes: "pcycles", "usecs" that | ||||||
contain the execution duration in terms of processor cycles and microseconds | ||||||
respectivaly, and an attribute `outputs`, which is a list of output file records | ||||||
whose syntax is identical to the input file records in the input file. | ||||||
A sample output JSON from running the Inception V3 model may look like | ||||||
``` | ||||||
{ | ||||||
"pcycles": 112965680178, | ||||||
"usecs": 79532302, | ||||||
"outputs": [ | ||||||
{ | ||||||
"file": "output0.dat", | ||||||
"shape": [1, 1001], | ||||||
"dtype": "float32" | ||||||
} | ||||||
] | ||||||
} | ||||||
``` | ||||||
|
||||||
The launcher does not perform any correctness verification. In order to verify | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Perhaps preface these 2 paragraphs with # Disclaimer or Future work There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Done. |
||||||
correctness, the user needs to copy the output files from the device and | ||||||
verify their contents. | ||||||
|
||||||
This launcher is intended for use with prototyping and does not utilize any | ||||||
performance acceleration, as such the measured performance may be very poor. |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Should we add a mention here of the Hexagon devices / Snapdragon SoCs that we expect the launcher to work on / have tested the launcher on?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Ah I think one can infer from l47:
one of v65, v66, v68
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
But explicitly stating the Snapdragon devices could be useful.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Done.