feat: add tensorrt_yolox package

perception/tensorrt_yolox/CMakeLists.txt

@@ -0,0 +1,189 @@
+cmake_minimum_required(VERSION 3.5)
+project(tensorrt_yolox)
+
+if(NOT CMAKE_CXX_STANDARD)
+  set(CMAKE_CXX_STANDARD 17)
+  set(CMAKE_CXX_STANDARD_REQUIRED ON)
+  set(CMAKE_CXX_EXTENSIONS OFF)
+endif()
+if(CMAKE_COMPILER_IS_GNUCXX OR CMAKE_CXX_COMPILER_ID MATCHES "Clang")
+  add_compile_options(-Wall -Wextra -Wpedantic -Werror)
+endif()
+
+find_package(ament_cmake_auto REQUIRED)
+ament_auto_find_build_dependencies()
+find_package(OpenCV REQUIRED)
+find_package(CUDA REQUIRED)
+find_package(CUDNN REQUIRED)
+find_package(TENSORRT REQUIRED)
+if(NOT (${CUDA_FOUND} AND ${CUDNN_FOUND} AND ${TENSORRT_FOUND}))
+  message(WARNING "cuda, cudnn, tensorrt libraries are not found")
+  return()
+endif()
+
+find_package(OpenMP)
+if(OpenMP_FOUND)
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
+endif()
+
+##########
+# Download pretrained model
+set(DATA_PATH "${CMAKE_CURRENT_SOURCE_DIR}/data")
+if(NOT EXISTS "${DATA_PATH}")
+  execute_process(COMMAND mkdir -p ${DATA_PATH})
+endif()
+function(download FILE_NAME FILE_HASH)
+  message(STATUS "Checking and downloading ${FILE_NAME}")
+  set(FILE_PATH ${DATA_PATH}/${FILE_NAME})
+  set(STATUS_CODE 0)
+  message(STATUS "start ${FILE_NAME}")
+  if(EXISTS ${FILE_PATH})
+    message(STATUS "found ${FILE_NAME}")
+    file(MD5 ${FILE_PATH} EXISTING_FILE_HASH)
+    if(${FILE_HASH} STREQUAL ${EXISTING_FILE_HASH})
+      message(STATUS "same ${FILE_NAME}")
+      message(STATUS "File already exists.")
+    else()
+      message(STATUS "diff ${FILE_NAME}")
+      message(STATUS "File hash changes. Downloading now ...")
+      file(DOWNLOAD https://awf.ml.dev.web.auto/perception/models/${FILE_NAME} ${FILE_PATH} STATUS DOWNLOAD_STATUS TIMEOUT 3600)
+      list(GET DOWNLOAD_STATUS 0 STATUS_CODE)
+      list(GET DOWNLOAD_STATUS 1 ERROR_MESSAGE)
+    endif()
+  else()
+    message(STATUS "not found ${FILE_NAME}")
+    message(STATUS "File doesn't exists. Downloading now ...")
+    file(DOWNLOAD https://awf.ml.dev.web.auto/perception/models/${FILE_NAME} ${FILE_PATH} STATUS DOWNLOAD_STATUS TIMEOUT 3600)
+    list(GET DOWNLOAD_STATUS 0 STATUS_CODE)
+    list(GET DOWNLOAD_STATUS 1 ERROR_MESSAGE)
+  endif()
+  if(${STATUS_CODE} EQUAL 0)
+    message(STATUS "Download completed successfully!")
+  else()
+    message(FATAL_ERROR "Error occurred during download: ${ERROR_MESSAGE}")
+  endif()
+endfunction()
+
+download(yolox-tiny.onnx 97028baf73ce55e115599c9c60651b08)
+download(label.txt 9ceadca8b72b6169ee6aabb861fe3e1e)
+
+# The following libraries, which are named cuda_utils and tensorrt_common,
+# are planned to be independent packages
+##########
+# [lib]　cuda_utils
+install(
+  DIRECTORY lib/include/cuda_utils/
+  DESTINATION include/cuda_utils
+)
+
+# ament_export_include_directories(lib/include)
+ament_export_dependencies(CUDA)
+
+##########
+# [lib] tensorrt_common
+cuda_add_library(tensorrt_common SHARED
+  lib/src/tensorrt_common/tensorrt_common.cpp
+)
+
+ament_target_dependencies(tensorrt_common
+  rclcpp
+)
+
+target_include_directories(tensorrt_common
+  PUBLIC
+  $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/lib/include>
+  $<INSTALL_INTERFACE:include>
+  ${TENSORRT_INCLUDE_DIRS}
+)
+
+target_link_libraries(tensorrt_common
+  ${TENSORRT_LIBRARIES}
+  stdc++fs
+)
+
+target_compile_definitions(tensorrt_common PRIVATE
+  TENSORRT_VERSION_MAJOR=${TENSORRT_VERSION_MAJOR}
+)
+
+list(APPEND tensorrt_common_LIBRARIES "tensorrt_common")
+
+# ament_export_include_directories(lib/include)
+ament_export_dependencies(cudnn_cmake_module)
+ament_export_dependencies(CUDNN)
+ament_export_dependencies(tensorrt_cmake_module)
+ament_export_dependencies(TENSORRT)
+
+include_directories(
+  lib/include
+)
+
+##########
+# tensorrt_yolox
+ament_auto_add_library(${PROJECT_NAME} SHARED
+  src/tensorrt_yolox.cpp
+)
+
+ament_target_dependencies(${PROJECT_NAME}
+  OpenCV
+)
+
+target_link_libraries(${PROJECT_NAME}
+  ${tensorrt_common_LIBRARIES}
+)
+
+target_compile_definitions(${PROJECT_NAME} PRIVATE
+  TENSORRT_VERSION_MAJOR=${TENSORRT_VERSION_MAJOR}
+)
+
+ament_auto_add_library(yolox_single_image_inferece_node SHARED
+  src/yolox_single_image_inference_node.cpp
+)
+
+ament_target_dependencies(yolox_single_image_inferece_node
+  OpenCV
+)
+
+target_link_libraries(yolox_single_image_inferece_node
+  ${PROJECT_NAME}
+  stdc++fs
+)
+
+target_compile_definitions(yolox_single_image_inferece_node PRIVATE
+  TENSORRT_VERSION_MAJOR=${TENSORRT_VERSION_MAJOR}
+)
+
+rclcpp_components_register_node(yolox_single_image_inferece_node
+  PLUGIN "tensorrt_yolox::YoloXSingleImageInferenceNode"
+  EXECUTABLE yolox_single_image_inferece
+)
+
+ament_auto_add_library(${PROJECT_NAME}_node SHARED
+  src/tensorrt_yolox_node.cpp
+)
+
+ament_target_dependencies(${PROJECT_NAME}_node
+  OpenCV
+)
+
+target_link_libraries(${PROJECT_NAME}_node
+  ${PROJECT_NAME}
+)
+
+target_compile_definitions(${PROJECT_NAME}_node PRIVATE
+  TENSORRT_VERSION_MAJOR=${TENSORRT_VERSION_MAJOR}
+)
+
+rclcpp_components_register_node(${PROJECT_NAME}_node
+  PLUGIN "tensorrt_yolox::TrtYoloXNode"
+  EXECUTABLE ${PROJECT_NAME}_node_exe
+)
+
+if(BUILD_TESTING)
+  find_package(ament_lint_auto REQUIRED)
+  ament_lint_auto_find_test_dependencies()
+endif()
+
+ament_auto_package(INSTALL_TO_SHARE
+  data
+)

perception/tensorrt_yolox/README.md

@@ -0,0 +1,142 @@
+# tensorrt_yolox
+
+## Purpose
+
+This package detects target objects e.g., cars, trucks, bicycles, and pedestrians on a image based on [YOLOX](https://github.com/Megvii-BaseDetection/YOLOX) model.
+
+## Inner-workings / Algorithms
+
+### Cite
+
+Zheng Ge, Songtao Liu, Feng Wang, Zeming Li, Jian Sun, "YOLOX: Exceeding YOLO Series in 2021", arXiv preprint arXiv:2107.08430, 2021 [[ref](https://arxiv.org/abs/2107.08430)]
+
+## Inputs / Outputs
+
+### Input
+
+| Name       | Type                | Description     |
+| ---------- | ------------------- | --------------- |
+| `in/image` | `sensor_msgs/Image` | The input image |
+
+### Output
+
+| Name          | Type                                               | Description                                        |
+| ------------- | -------------------------------------------------- | -------------------------------------------------- |
+| `out/objects` | `tier4_perception_msgs/DetectedObjectsWithFeature` | The detected objects with 2D bounding boxes        |
+| `out/image`   | `sensor_msgs/Image`                                | The image with 2D bounding boxes for visualization |
+
+## Parameters
+
+### Core Parameters
+
+| Name              | Type  | Default Value | Description                                                                           |
+| ----------------- | ----- | ------------- | ------------------------------------------------------------------------------------- |
+| `score_threshold` | float | 0.3           | If the objectness score is less than this value, the object is ignored in yolo layer. |
+| `nms_threshold`   | float | 0.7           | The IoU threshold for NMS method                                                      |
+
+**NOTE:** These two parameters are only valid for "plain" model (described later).
+
+### Node Parameters
+
+| Name         | Type   | Default Value | Description                                                        |
+| ------------ | ------ | ------------- | ------------------------------------------------------------------ |
+| `onnx_file`  | string | ""            | The onnx file name for yolo model                                  |
+| `label_file` | string | ""            | The label file with label names for detected objects written on it |
+| `mode`       | string | "fp32"        | The inference mode: "fp32", "fp16", "int8"                         |
+
+## Assumptions / Known limits
+
+The label contained in detected 2D bounding boxes (i.e., `out/objects`) will be either one of the followings:
+
+- CAR
+- PEDESTRIAN ("PERSON" will also be categorized as "PEDESTRIAN")
+- BUS
+- TRUCK
+- BICYCLE
+- MOTORCYCLE
+
+If other labels (case insensitive) are contained in the file specified via the `label_file` parameter,
+those are labeled as `UNKNOWN`, while detected rectangles are drawn in the visualization result (`out/image`).
+
+## Onnx model
+
+A sample model (named `yolox-tiny.onnx`) is downloaded automatically during the build process.
+To accelerate Non-maximum-suppression (NMS), which is one of the common post-process after object detection inference,
+`EfficientNMS_TRT` module is attached after the ordinal YOLOX (tiny) network.
+The `EfficientNMS_TRT` module contains fixed values for `score_threshold` and `nms_threshold` in it,
+hence these parameters are ignored when users specify ONNX models including this module.
+
+This package accepts both `EfficientNMS_TRT` attached ONNXs and [models published from the official YOLOX repository](https://github.com/Megvii-BaseDetection/YOLOX/tree/main/demo/ONNXRuntime#download-onnx-models) (we referred to them as "plain" models).
+
+All models are automatically converted to TensorRT format.
+These converted files will be saved in the same directory as specified ONNX files
+with `.engine` filename extension and reused from the next run.
+The conversion process may take a while (typically a few minutes) and the inference process is blocked
+until complete the conversion, so it will take some time until detection results are published on the first run.
+
+### Package acceptable model generation
+
+To convert users' own model that saved in PyTorch's `pth` format into ONNX,
+users can exploit the converter offered by the official repository.
+For the convenience, only procedures are described below.
+Please refer [the official document](https://github.com/Megvii-BaseDetection/YOLOX/tree/main/demo/ONNXRuntime#convert-your-model-to-onnx) for more detail.
+
+#### For plain models
+
+1. Install dependency
+
+   ```shell
+   git clone git@github.com:Megvii-BaseDetection/YOLOX.git
+   cd YOLOX
+   python3 setup.py develop --user
+   ```
+
+2. Convert pth into ONNX
+
+   ```shell
+   python3 tools/export_onnx.py \
+     --output-name YOUR_YOLOX.onnx \
+     -f YOUR_YOLOX.py \
+     -c YOUR_YOLOX.pth
+   ```
+
+#### For EfficientNMS_TRT embedded models
+
+1. Install dependency
+
+   ```shell
+   git clone git@github.com:Megvii-BaseDetection/YOLOX.git
+   cd YOLOX
+   python3 setup.py develop --user
+   pip3 install git+ssh://git@github.com/wep21/yolox_onnx_modifier.git --user
+   ```
+
+2. Convert pth into ONNX
+
+   ```shell
+   python3 tools/export_onnx.py \
+     --output-name YOUR_YOLOX.onnx \
+     -f YOUR_YOLOX.py \
+     -c YOUR_YOLOX.pth
+     --decode_in_inference
+   ```
+
+3. Embed `EfficientNMS_TRT` to the end of YOLOX
+
+   ```shell
+   yolox_onnx_modifier YOUR_YOLOX.onnx -o YOUR_YOLOX_WITH_NMS.onnx
+   ```
+
+## Label file
+
+A sample label file (named `label.txt`)is also downloaded automatically during the build process
+(**NOTE:** This file is incompatible with models that output labels for the COCO dataset (e.g., models from the official YOLOX repository)).
+
+This file represents the correspondence between class index (integer outputted from YOLOX network) and
+class label (strings making understanding easier). This package maps class IDs (incremented from 0)
+with labels according to the order in this file.
+
+## Reference repositories
+
+- <https://github.com/Megvii-BaseDetection/YOLOX>
+- <https://github.com/wep21/yolox_onnx_modifier>