diff --git a/src/backend_config.cc b/src/backend_config.cc index d69d97075..f384aa757 100644 --- a/src/backend_config.cc +++ b/src/backend_config.cc @@ -1,4 +1,4 @@ -// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -186,19 +186,6 @@ BackendConfigurationSpecializeBackendName( return Status::Success; } -Status -BackendConfigurationBackendLibraryName( - const std::string& backend_name, std::string* libname) -{ -#ifdef _WIN32 - *libname = "triton_" + backend_name + ".dll"; -#else - *libname = "libtriton_" + backend_name + ".so"; -#endif - - return Status::Success; -} - Status BackendConfigurationModelLoadGpuFraction( const triton::common::BackendCmdlineConfigMap& config_map, diff --git a/src/backend_config.h b/src/backend_config.h index acd2a0c21..c55acc160 100644 --- a/src/backend_config.h +++ b/src/backend_config.h @@ -1,4 +1,4 @@ -// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -64,10 +64,6 @@ Status BackendConfigurationSpecializeBackendName( const triton::common::BackendCmdlineConfigMap& config_map, const std::string& backend_name, std::string* specialized_name); -/// Return the shared library name for a backend. -Status BackendConfigurationBackendLibraryName( - const std::string& backend_name, std::string* libname); - /// Get GPU memory limit fraction for model loading /// from the backend configuration. Status BackendConfigurationModelLoadGpuFraction( diff --git a/src/backend_model.cc b/src/backend_model.cc index 8c58f9761..f1175b61b 100644 --- a/src/backend_model.cc +++ b/src/backend_model.cc @@ -1,4 +1,4 @@ -// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -66,9 +66,8 @@ TritonModel::Create( { model->reset(); - // The model configuration must specify a backend. The name of the - // corresponding shared library must be libtriton_.so. - std::string backend_name = model_config.backend(); + // The model configuration must specify a backend. + const std::string& backend_name = model_config.backend(); if (backend_name.empty()) { return Status( Status::Code::INVALID_ARG, @@ -103,55 +102,27 @@ TritonModel::Create( RETURN_IF_ERROR(BackendConfigurationSpecializeBackendName( backend_cmdline_config_map, backend_name, &specialized_backend_name)); - std::string backend_libname; - RETURN_IF_ERROR(BackendConfigurationBackendLibraryName( - specialized_backend_name, &backend_libname)); - - // Get the path to the backend shared library. Search path is - // version directory, model directory, global backend directory. - const auto localized_model_path = localized_model_dir->Path(); - const auto version_path = - JoinPath({localized_model_path, std::to_string(version)}); - const std::string global_path = - JoinPath({backend_dir, specialized_backend_name}); - std::vector search_paths = { - version_path, localized_model_path, global_path}; - - std::string backend_libdir; - std::string backend_libpath; - std::string python_runtime_modeldir; + bool is_python_based_backend = false; + std::vector search_paths = GetBackendLibrarySearchPaths( + model_path, version, backend_dir, backend_name); + std::string backend_libdir, backend_libpath; - RETURN_IF_ERROR(ResolveBackendPaths( - specialized_backend_name, backend_dir, model_config.name(), search_paths, - backend_libname, &backend_libdir, &backend_libpath, - &python_runtime_modeldir)); + RETURN_IF_ERROR(GetBackendLibraryProperties( + localized_model_dir->Path(), version, backend_dir, + specialized_backend_name, &model_config, &is_python_based_backend, + &search_paths, &backend_libdir, &backend_libpath)); - // `backend_libpath` always points to shared library path. - if (backend_libpath.empty()) { - return Status( - Status::Code::INVALID_ARG, - "unable to find '" + backend_libname + "' or '" + - specialized_backend_name + "/" + kPythonFilename + "' for model '" + - model_config.name() + "', searched: " + version_path + ", " + - model_path + ", " + global_path); + if (is_python_based_backend) { + RETURN_IF_ERROR(SetPythonBasedBackendExecutionEnvironment( + backend_libdir, &model_config)); } // Resolve the global backend configuration with the specific backend // configuration - bool is_python_based_backend = false; triton::common::BackendCmdlineConfig config; - if (!python_runtime_modeldir.empty()) { - // `backend_libdir` points to model.py for python backend based backends. - backend_libdir = python_runtime_modeldir; - is_python_based_backend = true; - // Python backend based backends use configs, specified for python backend - // in cmdline. - RETURN_IF_ERROR(ResolveBackendConfigs( - backend_cmdline_config_map, kPythonBackend, config)); - } else { - RETURN_IF_ERROR(ResolveBackendConfigs( - backend_cmdline_config_map, backend_name, config)); - } + RETURN_IF_ERROR(ResolveBackendConfigs( + backend_cmdline_config_map, + (is_python_based_backend ? kPythonBackend : backend_name), config)); RETURN_IF_ERROR(SetBackendConfigDefaults(config)); @@ -317,12 +288,140 @@ TritonModel::GetExecutionPolicy(const inference::ModelConfig& model_config) return Status::Success; } +std::vector +TritonModel::GetBackendLibrarySearchPaths( + const std::string& model_path, int64_t version, + const std::string& backend_dir, const std::string& backend_name) +{ + const auto version_path = JoinPath({model_path, std::to_string(version)}); + const auto backend_path = JoinPath({backend_dir, backend_name}); + std::vector search_paths = { + version_path, model_path, backend_path}; + return search_paths; +} + Status -TritonModel::LocateBackendLibrary( - const std::vector search_paths, +TritonModel::GetBackendLibraryProperties( + const std::string& model_path, int64_t version, + const std::string& backend_dir, const std::string& backend_name, + inference::ModelConfig* model_config, bool* is_python_based_backend, + std::vector* search_paths, std::string* backend_libdir, + std::string* backend_libpath) +{ + std::string python_based_backend_libdir; + std::string backend_libname = model_config->runtime(); + if (backend_libname.empty()) { + RETURN_IF_ERROR(GetBackendRuntimeLibraryName( + backend_dir, backend_name, *search_paths, &backend_libname, + backend_libdir, backend_libpath, is_python_based_backend)); + if (!*is_python_based_backend) { + // All variables are correctly set for C++ backends on initial search. + return Status::Success; + } + python_based_backend_libdir = *backend_libdir; + model_config->set_runtime(backend_libname); + } else { + *is_python_based_backend = backend_libname == kPythonFilename; + } + + std::string cpp_backend_libname = backend_libname; + if (*is_python_based_backend) { + // Set C++ library name to Python backend. + cpp_backend_libname = AssembleCPPRuntimeLibraryName(kPythonBackend); + // The search paths only contain locations related to the Python backend + // based backend, the global Python backend location has to be added. + search_paths->emplace_back(JoinPath({backend_dir, kPythonBackend})); + } + + RETURN_IF_ERROR(FindBackendLibraryPath( + *search_paths, cpp_backend_libname, backend_libdir, backend_libpath)); + if (backend_libpath->empty()) { + std::string search_paths_str = ""; + for (const auto& path : *search_paths) { + search_paths_str += "'" + path + "' "; + } + return Status( + Status::Code::INVALID_ARG, "unable to find backend library '" + + cpp_backend_libname + "' for model '" + + model_config->name() + + "', searched: " + search_paths_str); + } + if (IsChildPathEscapingParentPath( + *backend_libpath /* child_path */, + *backend_libdir /* parent_path */)) { + return Status( + Status::Code::INVALID_ARG, + "backend library name '" + cpp_backend_libname + + "' escapes backend directory '" + *backend_libdir + + "', for model '" + model_config->name() + + "', check model config runtime field"); + } + + // Both 'backend_libdir' and 'backend_libpath' are now pointing to the C++ + // backend library, 'backend_libdir' needs adjustment for Python based + // backend. + if (*is_python_based_backend) { + if (python_based_backend_libdir.empty()) { + python_based_backend_libdir = JoinPath({backend_dir, backend_name}); + // Make sure the library and its directory exist. + std::string path = + JoinPath({python_based_backend_libdir, kPythonFilename}); + bool path_exist; + RETURN_IF_ERROR(FileExists(path, &path_exist)); + if (!path_exist) { + return Status( + Status::Code::INVALID_ARG, + "unable to find Python backend based backend library '" + + backend_libname + "' for model '" + model_config->name() + + "', searched: '" + path + "'"); + } + } + *backend_libdir = python_based_backend_libdir; + } + + return Status::Success; +} + +Status +TritonModel::GetBackendRuntimeLibraryName( + const std::string& backend_dir, const std::string& backend_name, + const std::vector& search_paths, std::string* backend_libname, + std::string* backend_libdir, std::string* backend_libpath, + bool* is_python_based_backend) +{ + // Try C++ runtime + *backend_libname = AssembleCPPRuntimeLibraryName(backend_name); + RETURN_IF_ERROR(FindBackendLibraryPath( + search_paths, *backend_libname, backend_libdir, backend_libpath)); + if (!backend_libpath->empty()) { + *is_python_based_backend = false; + return Status::Success; + } + // Try Python runtime + std::vector python_search_paths = { + JoinPath({backend_dir, backend_name})}; + *backend_libname = kPythonFilename; + RETURN_IF_ERROR(FindBackendLibraryPath( + python_search_paths, *backend_libname, backend_libdir, backend_libpath)); + if (!backend_libpath->empty()) { + *is_python_based_backend = true; + return Status::Success; + } + // Cannot find runtime + return Status( + Status::Code::INVALID_ARG, + "unable to find backend library for backend '" + backend_name + + "', try specifying runtime on the model configuration."); +} + +Status +TritonModel::FindBackendLibraryPath( + const std::vector& search_paths, const std::string& backend_libname, std::string* backend_libdir, std::string* backend_libpath) { + backend_libpath->clear(); + for (const auto& path : search_paths) { const auto full_path = JoinPath({path, backend_libname}); bool exists = false; @@ -337,57 +436,14 @@ TritonModel::LocateBackendLibrary( return Status::Success; } -Status -TritonModel::ResolveBackendPaths( - const std::string& backend_name, const std::string& global_backend_dir, - const std::string& model_name, std::vector& search_paths, - const std::string& backend_libname, std::string* backend_libdir, - std::string* backend_libpath, std::string* python_runtime_modeldir) +std::string +TritonModel::AssembleCPPRuntimeLibraryName(const std::string& backend_name) { - // Look for shared library first - RETURN_IF_ERROR(LocateBackendLibrary( - search_paths, backend_libname, backend_libdir, backend_libpath)); - - if (!(*backend_libpath).empty()) { - *python_runtime_modeldir = ""; - return Status::Success; - } - - // If not found, then we are processing a python-based backend. - // We look for libtriton_python.so in python backend directory - // and model.py in provided custom backend's directory - std::string python_backend_dir = - JoinPath({global_backend_dir, kPythonBackend}); - bool is_dir; - RETURN_IF_ERROR(IsDirectory(python_backend_dir, &is_dir)); - if (!is_dir) { - return Status( - Status::Code::INVALID_ARG, "unable to find '" + global_backend_dir + - "/python', '" + backend_name + - "' requires python backend to operate."); - } - search_paths.emplace_back(python_backend_dir); - std::string runtime_model_path = - JoinPath({global_backend_dir, backend_name, kPythonFilename}); - bool exists; - RETURN_IF_ERROR(FileExists(runtime_model_path, &exists)); - if (!exists) { - return Status( - Status::Code::INVALID_ARG, - "unable to find '" + backend_libname + "' or '" + backend_name + "/" + - kPythonFilename + "' for model '" + model_name + "', in " + - JoinPath({global_backend_dir, backend_name})); - } - - *python_runtime_modeldir = JoinPath({global_backend_dir, backend_name}); - std::string python_backend_libname; - RETURN_IF_ERROR(BackendConfigurationBackendLibraryName( - kPythonBackend, &python_backend_libname)); - - RETURN_IF_ERROR(LocateBackendLibrary( - search_paths, python_backend_libname, backend_libdir, backend_libpath)); - - return Status::Success; +#ifdef _WIN32 + return "triton_" + backend_name + ".dll"; +#else + return "libtriton_" + backend_name + ".so"; +#endif } Status diff --git a/src/backend_model.h b/src/backend_model.h index ee2be504f..6ad1df257 100644 --- a/src/backend_model.h +++ b/src/backend_model.h @@ -1,4 +1,4 @@ -// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -195,25 +195,43 @@ class TritonModel : public Model { static Status SetBackendConfigDefaults( triton::common::BackendCmdlineConfig& config); - // Searches for backend_libname in provided search_paths. - // If found, stores backend directory in backend_libdir and - // backend path in backend_libpath. - static Status LocateBackendLibrary( - const std::vector search_paths, - const std::string& backend_libname, std::string* backend_libdir, + // Get the search paths to the backend shared library. + static std::vector GetBackendLibrarySearchPaths( + const std::string& model_path, int64_t version, + const std::string& backend_dir, const std::string& backend_name); + + // Get backend library directory and path, and search paths for the library + // and whether the backend is based on Python backend. The model configuration + // runtime field will be updated if left empty. + static Status GetBackendLibraryProperties( + const std::string& model_path, int64_t version, + const std::string& backend_dir, const std::string& backend_name, + inference::ModelConfig* model_config, bool* is_python_based_backend, + std::vector* search_paths, std::string* backend_libdir, std::string* backend_libpath); - // For a given backend (`backend_name`), looks for backend directory and - // location for the shared library, used by the backend. Returns: - // `backend_libdir` returns directory, where shared library (.so) is stored, - // `backend_libpath` returns the full path to .so, - // `python_runtime_modeldir` is set to empty string for c++ backends and - // returns directory, where model.py is stored. - static Status ResolveBackendPaths( - const std::string& backend_name, const std::string& global_backend_dir, - const std::string& model_name, std::vector& search_paths, + // Get 'backend_libname', 'backend_libdir', 'backend_libpath' and + // 'is_python_based_backend' by searching for different possible backend + // library names on 'search_paths'. Searching for Python based backend + // runtime is limited to 'backend_dir'. + static Status GetBackendRuntimeLibraryName( + const std::string& backend_dir, const std::string& backend_name, + const std::vector& search_paths, + std::string* backend_libname, std::string* backend_libdir, + std::string* backend_libpath, bool* is_python_based_backend); + + // Search for 'backend_libname' on 'search_paths'. If found, the matching + // search path will be stored in 'backend_libdir' and the backend library path + // will be stored in 'backend_libpath'. If not found, 'backend_libpath' will + // be set to empty. + static Status FindBackendLibraryPath( + const std::vector& search_paths, const std::string& backend_libname, std::string* backend_libdir, - std::string* backend_libpath, std::string* python_runtime_modeldir); + std::string* backend_libpath); + + // Assemble the C++ runtime library name. + static std::string AssembleCPPRuntimeLibraryName( + const std::string& backend_name); // Clear library handles. void ClearHandles(); diff --git a/src/filesystem/api.cc b/src/filesystem/api.cc index 2734eaf48..b81378ada 100644 --- a/src/filesystem/api.cc +++ b/src/filesystem/api.cc @@ -1,4 +1,4 @@ -// Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -45,6 +45,7 @@ #include #include +#include #include namespace triton { namespace core { @@ -398,6 +399,20 @@ IsAbsolutePath(const std::string& path) return !path.empty() && (path[0] == '/'); } +bool +IsChildPathEscapingParentPath( + const std::string& child_path, const std::string& parent_path) +{ + // Can use std::filesystem over boost in C++17. + const std::string absolute_child_path = + boost::filesystem::weakly_canonical(child_path).string(); + const std::string absolute_parent_path = + boost::filesystem::canonical(parent_path).string(); + // Can use starts_with() over rfind() in C++20. + bool is_escape = absolute_child_path.rfind(absolute_parent_path, 0) != 0; + return is_escape; +} + std::string JoinPath(std::initializer_list segments) { diff --git a/src/filesystem/api.h b/src/filesystem/api.h index 8ee0b373e..67bb53e1f 100644 --- a/src/filesystem/api.h +++ b/src/filesystem/api.h @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved. +// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -78,6 +78,14 @@ class LocalizedPath { /// \return true if absolute path, false if relative path. bool IsAbsolutePath(const std::string& path); +/// Check if the child path escapes from its parent path. +/// \param child_path The child path. +/// \param parent_path The parent path. The path must exist. +/// \return true if the child path escapes from its parent path, false if the +/// child path is within its parent path. +bool IsChildPathEscapingParentPath( + const std::string& child_path, const std::string& parent_path); + /// Join path segments into a longer path /// \param segments The path segments. /// \return the path formed by joining the segments. diff --git a/src/model_config_utils.cc b/src/model_config_utils.cc index 43ecd661e..d24a3b3e1 100644 --- a/src/model_config_utils.cc +++ b/src/model_config_utils.cc @@ -1,4 +1,4 @@ -// Copyright 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -933,6 +933,25 @@ LocalizePythonBackendExecutionEnvironmentPath( return Status::Success; } +Status +SetPythonBasedBackendExecutionEnvironment( + const std::string& backend_libdir, inference::ModelConfig* model_config) +{ + if (!model_config->parameters().contains("EXECUTION_ENV_PATH")) { + std::string env_name = "pb_exec_env_" + model_config->runtime() + ".tar.gz"; + std::string env_path = JoinPath({backend_libdir, std::move(env_name)}); + bool env_path_exist; + RETURN_IF_ERROR(FileExists(env_path, &env_path_exist)); + if (env_path_exist) { + inference::ModelParameter model_param; + model_param.set_string_value(env_path); + (*model_config->mutable_parameters())["EXECUTION_ENV_PATH"] = + std::move(model_param); + } + } + return Status::Success; +} + Status SetDefaultInstanceCount( inference::ModelInstanceGroup* group, const std::string& backend) @@ -1106,7 +1125,7 @@ AutoCompleteBackendFields( return Status::Success; } - // PyTorch (TorchScript, LibTorch) + // PyTorch if (config->backend().empty()) { if ((config->platform() == kPyTorchLibTorchPlatform) || (config->default_model_filename() == kPyTorchLibTorchFilename)) { @@ -1127,9 +1146,11 @@ AutoCompleteBackendFields( } if (config->backend() == kPyTorchBackend) { if (config->platform().empty()) { + // do not introduce new platforms, new runtimes may ignore this field. config->set_platform(kPyTorchLibTorchPlatform); } - if (config->default_model_filename().empty()) { + if (config->runtime() != kPythonFilename && + config->default_model_filename().empty()) { config->set_default_model_filename(kPyTorchLibTorchFilename); } return Status::Success; diff --git a/src/model_config_utils.h b/src/model_config_utils.h index 018f4e7a3..8bd9af600 100644 --- a/src/model_config_utils.h +++ b/src/model_config_utils.h @@ -1,4 +1,4 @@ -// Copyright 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions @@ -129,6 +129,15 @@ Status LocalizePythonBackendExecutionEnvironmentPath( const std::string& model_path, inference::ModelConfig* config, std::shared_ptr* localized_model_dir); +/// Set execution environments for Python based backends, if the execution +/// environments is not already specified and the execution environment file +/// '/pb_exec_.tar.gz' exists. +/// \param backend_libdir The backend runtime library directory path. +/// \param model_config The model configuration. +/// \return The error status. +Status SetPythonBasedBackendExecutionEnvironment( + const std::string& backend_libdir, inference::ModelConfig* model_config); + /// Auto-complete the instance count based on instance kind and backend name. /// \param group The instance group to set the count for. /// \param backend The backend name to check against. diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt index 2816d52b9..a0a3177d2 100644 --- a/src/test/CMakeLists.txt +++ b/src/test/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -126,6 +126,7 @@ if(${TRITON_ENABLE_GPU}) ${CMAKE_CURRENT_SOURCE_DIR}/../../include ${GTEST_INCLUDE_DIRS} ${CNMEM_PATH}/include + ${Boost_INCLUDE_DIRS} ) target_compile_definitions( @@ -151,6 +152,7 @@ if(${TRITON_ENABLE_GPU}) protobuf::libprotobuf ${CNMEM_LIBRARY} CUDA::cudart + Boost::filesystem ) if (NOT WIN32) @@ -417,6 +419,7 @@ target_include_directories( ${CMAKE_CURRENT_SOURCE_DIR}/.. ${CMAKE_CURRENT_SOURCE_DIR}/../../include ${GTEST_INCLUDE_DIRS} + ${Boost_INCLUDE_DIRS} ) target_link_libraries( @@ -430,6 +433,7 @@ target_link_libraries( GTest::gtest GTest::gtest_main protobuf::libprotobuf + Boost::filesystem ) install(