IWYU CI integration for Morpheus (#287)

* Add IWYU check to CI
* Fix includes to pass
* Rename `deserialization.hpp` to `deserialize.hpp` such that the filename matches the associated cpp & python file names. This also causes IWYU to identify the header as being associated.

Authors:
  - David Gardner (https://github.com/dagardner-nv)

Approvers:
  - Michael Demoret (https://github.com/mdemoret-nv)

URL: #287

CMakeLists.txt

@@ -29,6 +29,7 @@ option(MORPHEUS_USE_CCACHE "Enable caching compilation results with ccache" OFF)
 option(MORPHEUS_USE_CLANG_TIDY "Enable running clang-tidy as part of the build process" OFF)
 option(MORPHEUS_USE_CONDA "Enables finding dependencies via conda instead of vcpkg.
   Note: This will disable vcpkg. All dependencies must be installed first in the conda environment" OFF)
+option(MORPHEUS_USE_IWYU "Enable running include-what-you-use as part of the build process" OFF)
 set(MORPHEUS_PY_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/wheel" CACHE STRING "Location to install the python directory")
 
 set(MORPHEUS_CACHE_DIR "${CMAKE_SOURCE_DIR}/.cache" CACHE PATH "Directory to contain all CPM and CCache data")
@@ -74,6 +75,9 @@ include(cmake/dependencies.cmake)
 # - Post dependencies setup --------
 include(cmake/setup_compiler.cmake)
 
+# Setup IWYU if enabled
+include(cmake/setup_iwyu.cmake)
+
 # To make it easier for CI to find output files, set the default executable suffix to .x if not set
 if("${CMAKE_EXECUTABLE_SUFFIX}" STREQUAL "")
   set(CMAKE_EXECUTABLE_SUFFIX ".x")

ci/iwyu/mappings.imp

@@ -37,11 +37,55 @@
 # Protobuf
 { "include": [ "<google/protobuf/repeated_ptr_field.h>", private, "<google/protobuf/repeated_field.h>", "public" ] },
 
+# rxcpp
+# Hide includes that are exported by <rxcpp/rx.hpp>
+{ "include": [ "\"rx-includes.hpp\"", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rxcpp/rx-util.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "\"rxcpp/rx-util.hpp\"", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rxcpp/rx-predef.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "\"rxcpp/rx-predef.hpp\"", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rxcpp/rx-subscription.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "\"rxcpp/rx-subscription.hpp\"", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rx-observable.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rx-observer.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rxcpp/rx-observer.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "\"rxcpp/rx-observer.hpp\"", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rx-operators.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rx-predef.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rxcpp/rx-scheduler.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "\"rxcpp/rx-scheduler.hpp\"", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rxcpp/rx-subscriber.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "\"rxcpp/rx-subscriber.hpp\"", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rx-subscriber.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rxcpp/rx-notification.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "\"rxcpp/rx-notification.hpp\"", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rxcpp/rx-coordination.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "\"rxcpp/rx-coordination.hpp\"", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rxcpp/rx-sources.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "\"rxcpp/rx-sources.hpp\"", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rxcpp/rx-subjects.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "\"rxcpp/rx-subjects.hpp\"", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rxcpp/rx-operators.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "\"rxcpp/rx-operators.hpp\"", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rxcpp/rx-observable.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "\"rxcpp/rx-observable.hpp\"", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rxcpp/rx-connectable_observable.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "\"rxcpp/rx-connectable_observable.hpp\"", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<rxcpp/rx-grouped_observable.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "\"rxcpp/rx-grouped_observable.hpp\"", private, "<rxcpp/rx.hpp>", "public" ] },
+{ "include": [ "<sources/rx-iterate.hpp>", private, "<rxcpp/rx.hpp>", "public" ] },
+
+#Triton Client
+{ "include": ["\"common.h\"", "private", "<http_client.h>", "public"] },
+
 ## Symbol mappings
 # stdlib
 { "symbol": ["__gnu_cxx::__enable_if<true, double>::__type", "private", "<cmath>", "public"] },
 { "symbol": ["std::__success_type<std::chrono::duration<long, std::ratio<1, 1000000000>>>::type" , "private", "<chrono>", "public"] },
 { "symbol": ["__cxxabiv1::__forced_unwind", "private", "<future>", "public"] },
+{ "symbol": ["std::filesystem", "private", "<filesystem>", "public"] },
+{ "symbol": ["size_t", , "private", "<cstddef>", "public"] },
+{ "symbol": ["std::size_t", , "private", "<cstddef>", "public"] },
 
 # boost
 { "symbol": ["__forced_unwind", "private", "<boost/fiber/all.hpp>", "public"] },
@@ -66,20 +110,27 @@
 { "symbol": ["pybind11", "private", "<pybind11/pybind11.h>", "public"] },
 { "symbol": ["pybind11", "private", "<pybind11/pytypes.h>", "public"] },
 
+{ "symbol": ["_Py_IsFinalizing", "private", "<pybind11/pybind11.h>", "public"] },
 { "symbol": ["PYBIND11_MODULE", "private", "<pybind11/pybind11.h>", "public"] },
-{ "symbol": ["PySequence_GetItem", "private", "<pybind11/pybind11.h>", "public"] },
 { "symbol": ["PyExc_ImportError", "private", "<pybind11/pybind11.h>", "public"] },
+{ "symbol": ["PyErr_SetNone", "private", "<pybind11/pybind11.h>", "public"] },
 { "symbol": ["PyErr_SetObject", "private", "<pybind11/pybind11.h>", "public"] },
 { "symbol": ["PyExc_StopIteration", "private", "<pybind11/pybind11.h>", "public"] },
-{ "symbol": ["_Py_IsFinalizing", "private", "<pybind11/pybind11.h>", "public"] },
-{ "symbol": ["pybind11::detail::str_attr_accessor", "private", "<pybind11/pybind11.h>", "public"] },
+{ "symbol": ["PyGILState_Check", "private", "<pybind11/gil.h>", "public"] },
+{ "symbol": ["PyObject", "private", "<pybind11/pybind11.h>", "public"] },
+{ "symbol": ["PySequence_GetItem", "private", "<pybind11/pybind11.h>", "public"] },
+{ "symbol": ["pybind11::arg", "private", "<pybind11/pybind11.h>", "public"] },
+{ "symbol": ["pybind11::detail::get_type_info", "private", "<pybind11/cast.h>", "public"] },
+{ "symbol": ["pybind11::detail::key_error", "private", "<pybind11/pytypes.h>", "public"] },
 { "symbol": ["pybind11::detail::overload_cast_impl", "private", "<pybind11/pybind11.h>", "public"] },
+{ "symbol": ["pybind11::detail::str_attr_accessor", "private", "<pybind11/pybind11.h>", "public"] },
+{ "symbol": ["pybind11::key_error", "private", "<pybind11/pytypes.h>", "public"] },
 { "symbol": ["pybind11::overload_cast", "private", "<pybind11/pybind11.h>", "public"] },
-{ "symbol": ["pybind11::stop_iteration", "private", "<pybind11/pybind11.h>", "public"] },
 { "symbol": ["pybind11::return_value_policy", "private", "<pybind11/pybind11.h>", "public"] },
+{ "symbol": ["pybind11::return_value_policy::move", "private", "<pybind11/pybind11.h>", "public"] },
 { "symbol": ["pybind11::return_value_policy::reference_internal", "private", "<pybind11/pybind11.h>", "public"] },
-{ "symbol": ["pybind11::detail::get_type_info", "private", "<pybind11/cast.h>", "public"] },
-{ "symbol": ["PyGILState_Check", "private", "<pybind11/gil.h>", "public"] },
+{ "symbol": ["pybind11::size_t", "private", "<pybind11/pytypes.h>", "public"] },
+{ "symbol": ["pybind11::stop_iteration", "private", "<pybind11/pybind11.h>", "public"] },
 
 # spdlog
 { "symbol": ["spdlog::details::file_helper::~file_helper", "private", "<spdlog/sinks/basic_file_sink.h>", "public"] },

ci/scripts/cpp_checks.sh

@@ -80,7 +80,7 @@ if [[ -n "${MORPHEUS_MODIFIED_FILES}" ]]; then
 
    # Include What You Use
    if [[ "${SKIP_IWYU}" == "" ]]; then
-      IWYU_DIRS="benchmarks examples python src tools"
+      IWYU_DIRS="morpheus"
       NUM_PROC=$(get_num_proc)
       IWYU_OUTPUT=`${IWYU_TOOL} -p ${BUILD_DIR} -j ${NUM_PROC} ${IWYU_DIRS} 2>&1`
       IWYU_RETVAL=$?

ci/scripts/jenkins/checks.sh

@@ -17,11 +17,12 @@
 set -e
 
 source ${WORKSPACE}/ci/scripts/jenkins/common.sh
+export IWYU_DIR="${WORKSPACE_TMP}/iwyu"
 
 fetch_base_branch
 
 gpuci_logger "Creating conda env"
-rm -rf ${MORPHEUS_ROOT}/.cache/ ${MORPHEUS_ROOT}/build/
+rm -rf ${MORPHEUS_ROOT}/.cache/ ${MORPHEUS_ROOT}/build/ ${IWYU_DIR}
 conda config --add pkgs_dirs /opt/conda/pkgs
 conda config --env --add channels conda-forge
 conda config --env --set channel_alias ${CONDA_CHANNEL_ALIAS:-"https://conda.anaconda.org"}
@@ -33,6 +34,21 @@ mamba env update -q -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda${CUDA_VER
 
 show_conda_info
 
+gpuci_logger "Installing IWYU"
+git clone https://github.com/include-what-you-use/include-what-you-use.git ${IWYU_DIR}
+pushd ${IWYU_DIR}
+git checkout clang_12
+cmake -G Ninja \
+    -DCMAKE_PREFIX_PATH=$(llvm-config --cmakedir) \
+    -DCMAKE_C_COMPILER=$(which clang) \
+    -DCMAKE_CXX_COMPILER=$(which clang++) \
+    -DCMAKE_INSTALL_PREFIX=${CONDA_PREFIX} \
+    .
+
+cmake --build . --parallel ${PARALLEL_LEVEL} --target install
+
+popd
+
 gpuci_logger "Runing Python style checks"
 ${MORPHEUS_ROOT}/ci/scripts/python_checks.sh
 
@@ -46,7 +62,7 @@ gpuci_logger "sccache usage for source build:"
 sccache --show-stats
 
 gpuci_logger "Runing C++ style checks"
-SKIP_IWYU=1 ${MORPHEUS_ROOT}/ci/scripts/cpp_checks.sh
+${MORPHEUS_ROOT}/ci/scripts/cpp_checks.sh
 
 gpuci_logger "Checking copyright headers"
 python ${MORPHEUS_ROOT}/ci/scripts/copyright.py --verify-apache-v2 --git-diff-commits ${CHANGE_TARGET} ${GIT_COMMIT}

ci/scripts/run_iwyu_for_ci.sh

@@ -25,5 +25,5 @@ ${IWYU_TOOL_PY:-iwyu_tool.py} "$@" -- \
    -Xiwyu --no_fwd_decls \
    -Xiwyu --quoted_includes_first \
    -Xiwyu --cxx17ns \
-   -Xiwyu --no_comments \
+   -Xiwyu --max_line_length=120 \
    --driver-mode=g++

cmake/run_iwyu.sh.in

@@ -0,0 +1,19 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Allows running ccache with options inside of CMake. CMake does not work well
+# with setting variables before calling a command
+${MORPHEUS_IWYU_PROGRAM} ${MORPHEUS_IWYU_OPTIONS} "$@"

cmake/setup_iwyu.cmake

@@ -0,0 +1,61 @@
+#=============================================================================
+# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#=============================================================================
+
+
+function(configure_iwyu)
+  list(APPEND CMAKE_MESSAGE_CONTEXT "iwyu")
+
+  set(MORPHEUS_IWYU_VERBOSITY "1" CACHE STRING "Set verbosity level for include-what-you-use, 1 is default, 1 only shows recomendations and 11+ prints everything")
+
+  find_program(MORPHEUS_IWYU_PROGRAM "include-what-you-use")
+
+  if(MORPHEUS_IWYU_PROGRAM)
+    set(MORPHEUS_IWYU_OPTIONS
+        -Xiwyu; --mapping_file=${PROJECT_SOURCE_DIR}/ci/iwyu/mappings.imp;
+        -Xiwyu; --max_line_length=120;
+        -Xiwyu; --verbose=${MORPHEUS_IWYU_VERBOSITY};
+        -Xiwyu; --no_fwd_decls;
+        -Xiwyu; --quoted_includes_first;
+        -Xiwyu; --cxx17ns;)
+
+    # Convert these to space separated arguments
+    string(REPLACE ";" " " MORPHEUS_IWYU_OPTIONS "${MORPHEUS_IWYU_OPTIONS}")
+
+    message(STATUS "Enabling include-what-you-use for Morpheus targets")
+
+    set(IWYU_WRAPPER "${CMAKE_CURRENT_BINARY_DIR}/run_iwyu.sh")
+
+    # Make a ccache runner file with the necessary settings. ccache must already be configured
+    configure_file("${CMAKE_CURRENT_SOURCE_DIR}/cmake/run_iwyu.sh.in" "${IWYU_WRAPPER}")
+
+    if(MORPHEUS_USE_CCACHE)
+      set(CMAKE_C_INCLUDE_WHAT_YOU_USE "${CMAKE_CURRENT_BINARY_DIR}/run_ccache_prefix.sh;${IWYU_WRAPPER};${CMAKE_C_COMPILER}" PARENT_SCOPE)
+      set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE "${CMAKE_CURRENT_BINARY_DIR}/run_ccache_prefix.sh;${IWYU_WRAPPER};${CMAKE_CXX_COMPILER}" PARENT_SCOPE)
+    else()
+      set(CMAKE_C_INCLUDE_WHAT_YOU_USE "${IWYU_WRAPPER}" PARENT_SCOPE)
+      set(CMAKE_CXX_INCLUDE_WHAT_YOU_USE "${IWYU_WRAPPER}" PARENT_SCOPE)
+    endif()
+
+  else()
+    message(WARNING "IWYU option MORPHEUS_USE_IWYU is enabled but the include-what-you-use was not found. Check iwyu installation and add the iwyu bin dir to your PATH variable.")
+  endif(MORPHEUS_IWYU_PROGRAM)
+endfunction()
+
+# Configure IWYU if requested
+if(MORPHEUS_USE_IWYU)
+  configure_iwyu()
+endif(MORPHEUS_USE_IWYU)

docker/conda/environments/cuda11.5_ci.yml

@@ -21,9 +21,12 @@ dependencies:
     - bash=5.1.16 # bash >=4.4 is necessary for morpheus cli auto-complete tests to pass.
     - clang=12
     - clang-tools=12
+    - clangdev=12
+    - clangxx=12
     - conda-pack=0.7
     - git-lfs=3.2
     - git>=2.35.3 # Needed for wildcards on safe.directory
+    - llvmdev=12
     - pip
     - pkg-config=0.29
     - sccache=0.3

morpheus/_lib/cmake/libraries/cuda_utils.cmake

@@ -16,15 +16,45 @@ list(APPEND CMAKE_MESSAGE_CONTEXT "cuda_utils")
 
 find_package(pybind11 REQUIRED)
 
+# Place the two cuda sources in their own target and disable IWYU for that target.
+add_library(cuda_utils_objs
+OBJECT
+      ${MORPHEUS_LIB_ROOT}/src/utilities/matx_util.cu
+      ${MORPHEUS_LIB_ROOT}/src/utilities/type_util.cu
+)
+
+
+set_target_properties(
+  cuda_utils_objs
+    PROPERTIES
+      C_INCLUDE_WHAT_YOU_USE ""
+      CXX_INCLUDE_WHAT_YOU_USE ""
+      EXPORT_COMPILE_COMMANDS OFF
+)
+
+target_include_directories(cuda_utils_objs
+    PUBLIC
+      "${MORPHEUS_LIB_ROOT}/include"
+      cudf::cudf
+      matx::matx
+      srf::pysrf
+)
+
+target_link_libraries(cuda_utils_objs
+    PUBLIC
+      cudf::cudf
+      matx::matx
+      srf::pysrf
+)
+
 add_library(cuda_utils
     SHARED
+    $<TARGET_OBJECTS:cuda_utils_objs>
       ${MORPHEUS_LIB_ROOT}/src/objects/dev_mem_info.cpp
       ${MORPHEUS_LIB_ROOT}/src/objects/table_info.cpp
       ${MORPHEUS_LIB_ROOT}/src/objects/tensor_object.cpp
-      ${MORPHEUS_LIB_ROOT}/src/utilities/matx_util.cu
       ${MORPHEUS_LIB_ROOT}/src/utilities/tensor_util.cpp
       ${MORPHEUS_LIB_ROOT}/src/utilities/type_util_detail.cpp
-      ${MORPHEUS_LIB_ROOT}/src/utilities/type_util.cu
 )
 
 target_include_directories(cuda_utils

morpheus/_lib/cmake/libraries/cudf_helpers.cmake

@@ -33,6 +33,17 @@ morpheus_add_cython_libraries(
 # The `style_checks` target allows these to be generated without a full build of Morpheus.
 add_dependencies(style_checks ${cudf_helpers_target})
 
+
+# Disable clang-tidy and IWYU for cython generated code
+set_target_properties(
+  ${cudf_helpers_target}
+    PROPERTIES
+      CXX_CLANG_TIDY ""
+      C_INCLUDE_WHAT_YOU_USE ""
+      CXX_INCLUDE_WHAT_YOU_USE ""
+      EXPORT_COMPILE_COMMANDS OFF
+)
+
 if (MORPHEUS_PYTHON_INPLACE_BUILD)
   inplace_build_copy(${cudf_helpers_target} ${MORPHEUS_LIB_ROOT})
 endif()

morpheus/_lib/include/morpheus/io/serializers.hpp

@@ -21,7 +21,6 @@
 
 #include <ostream>
 #include <string>
-#include <vector>
 
 namespace morpheus {
 

morpheus/_lib/include/morpheus/messages/memory/inference_memory.hpp

@@ -18,15 +18,11 @@
 #pragma once
 
 #include "morpheus/messages/memory/tensor_memory.hpp"
-#include "morpheus/objects/tensor.hpp"
-#include "morpheus/objects/tensor_object.hpp"
 
 #include <cstddef>
-#include <map>
 #include <string>
 
 namespace morpheus {
-
 /**
  * TODO(Documentation)
  */

morpheus/_lib/include/morpheus/messages/memory/inference_memory_fil.hpp

@@ -18,15 +18,12 @@
 #pragma once
 
 #include "morpheus/messages/memory/inference_memory.hpp"
-#include "morpheus/objects/python_data_table.hpp"
-#include "morpheus/objects/tensor.hpp"
 #include "morpheus/objects/tensor_object.hpp"
 
-#include <cudf/io/types.hpp>
-#include <pybind11/pybind11.h>
+#include <cudf/types.hpp>      // for size_type
+#include <pybind11/pytypes.h>  // for object
 
 #include <cstddef>
-#include <filesystem>
 #include <memory>
 #include <string>
 

morpheus/_lib/include/morpheus/messages/memory/inference_memory_nlp.hpp

@@ -18,9 +18,9 @@
 #pragma once
 
 #include "morpheus/messages/memory/inference_memory.hpp"
-#include "morpheus/objects/tensor.hpp"
 #include "morpheus/objects/tensor_object.hpp"
 
+#include <cudf/types.hpp>  // for size_type
 #include <pybind11/pytypes.h>
 
 #include <cstddef>

morpheus/_lib/include/morpheus/messages/memory/response_memory.hpp

@@ -18,13 +18,12 @@
 #pragma once
 
 #include "morpheus/messages/memory/tensor_memory.hpp"
-#include "morpheus/objects/tensor.hpp"
+#include "morpheus/objects/tensor_object.hpp"  // for TensorObject
 
-#include <cudf/io/types.hpp>
 #include <pybind11/pytypes.h>
 
+#include <cstddef>  // for size_t
 #include <string>
-#include <vector>
 
 namespace morpheus {
 /****** Component public implementations *******************/