From 72a197f462b1aa05be7a3d4bcbc25c342dbd9573 Mon Sep 17 00:00:00 2001
From: kthui <18255193+kthui@users.noreply.github.com>
Date: Wed, 1 Nov 2023 17:46:36 -0700
Subject: [PATCH 01/14] Add autocomplete for runtime model config field

---
 src/constants.h           |  2 +
 src/model_config_utils.cc | 77 +++++++++++++++++++++++++++++++++++++--
 src/model_config_utils.h  | 21 +++++++++--
 3 files changed, 94 insertions(+), 6 deletions(-)

diff --git a/src/constants.h b/src/constants.h
index da6b76706..a7c7a68cf 100644
--- a/src/constants.h
+++ b/src/constants.h
@@ -60,6 +60,8 @@ constexpr char kPyTorchBackend[] = "pytorch";
 constexpr char kPythonFilename[] = "model.py";
 constexpr char kPythonBackend[] = "python";
 
+constexpr char kVLLMBackend[] = "vllm";
+
 #ifdef TRITON_ENABLE_ENSEMBLE
 constexpr char kEnsemblePlatform[] = "ensemble";
 #endif  // TRITON_ENABLE_ENSEMBLE
diff --git a/src/model_config_utils.cc b/src/model_config_utils.cc
index 43ecd661e..e3ae180ca 100644
--- a/src/model_config_utils.cc
+++ b/src/model_config_utils.cc
@@ -979,7 +979,7 @@ AutoCompleteBackendFields(
     config->set_name(model_name);
   }
 
-  // Trying to fill the 'backend', 'default_model_filename' field.
+  // Trying to fill the 'backend', 'default_model_filename' and 'runtime' field.
 
   // TensorFlow
   // For TF backend, the platform is required
@@ -1016,7 +1016,7 @@ AutoCompleteBackendFields(
     }
   }
 
-  // Fill 'backend' and 'default_model_filename' if missing
+  // Fill 'backend', 'default_model_filename' and 'runtime' if missing
   if ((config->platform() == kTensorFlowSavedModelPlatform) ||
       (config->platform() == kTensorFlowGraphDefPlatform)) {
     if (config->backend().empty()) {
@@ -1029,6 +1029,8 @@ AutoCompleteBackendFields(
         config->set_default_model_filename(kTensorFlowGraphDefFilename);
       }
     }
+    RETURN_IF_ERROR(
+        AutoCompleteBackendRuntimeField(RuntimeType::RUNTIME_TYPE_CPP, config));
     return Status::Success;
   }
 
@@ -1058,6 +1060,8 @@ AutoCompleteBackendFields(
     if (config->default_model_filename().empty()) {
       config->set_default_model_filename(kTensorRTPlanFilename);
     }
+    RETURN_IF_ERROR(
+        AutoCompleteBackendRuntimeField(RuntimeType::RUNTIME_TYPE_CPP, config));
     return Status::Success;
   }
 
@@ -1083,6 +1087,8 @@ AutoCompleteBackendFields(
     if (config->default_model_filename().empty()) {
       config->set_default_model_filename(kOnnxRuntimeOnnxFilename);
     }
+    RETURN_IF_ERROR(
+        AutoCompleteBackendRuntimeField(RuntimeType::RUNTIME_TYPE_CPP, config));
     return Status::Success;
   }
 
@@ -1103,10 +1109,12 @@ AutoCompleteBackendFields(
     if (config->default_model_filename().empty()) {
       config->set_default_model_filename(kOpenVINORuntimeOpenVINOFilename);
     }
+    RETURN_IF_ERROR(
+        AutoCompleteBackendRuntimeField(RuntimeType::RUNTIME_TYPE_CPP, config));
     return Status::Success;
   }
 
-  // PyTorch (TorchScript, LibTorch)
+  // PyTorch
   if (config->backend().empty()) {
     if ((config->platform() == kPyTorchLibTorchPlatform) ||
         (config->default_model_filename() == kPyTorchLibTorchFilename)) {
@@ -1132,6 +1140,8 @@ AutoCompleteBackendFields(
     if (config->default_model_filename().empty()) {
       config->set_default_model_filename(kPyTorchLibTorchFilename);
     }
+    RETURN_IF_ERROR(AutoCompleteBackendRuntimeField(
+        RuntimeType::RUNTIME_TYPE_UNKNOWN, config));
     return Status::Success;
   }
 
@@ -1152,6 +1162,18 @@ AutoCompleteBackendFields(
     if (config->default_model_filename().empty()) {
       config->set_default_model_filename(kPythonFilename);
     }
+    RETURN_IF_ERROR(
+        AutoCompleteBackendRuntimeField(RuntimeType::RUNTIME_TYPE_CPP, config));
+    return Status::Success;
+  }
+
+  // vLLM
+  if (config->backend() == kVLLMBackend) {
+    if (config->default_model_filename().empty()) {
+      config->set_default_model_filename(kPythonFilename);
+    }
+    RETURN_IF_ERROR(AutoCompleteBackendRuntimeField(
+        RuntimeType::RUNTIME_TYPE_PYTHON, config));
     return Status::Success;
   }
 
@@ -1180,9 +1202,58 @@ AutoCompleteBackendFields(
     config->set_backend(backend_name);
     config->set_default_model_filename(
         (std::string("model.") + backend_name).c_str());
+    RETURN_IF_ERROR(AutoCompleteBackendRuntimeField(
+        RuntimeType::RUNTIME_TYPE_UNKNOWN, config));
     return Status::Success;
   }
 
+  RETURN_IF_ERROR(AutoCompleteBackendRuntimeField(
+      RuntimeType::RUNTIME_TYPE_UNKNOWN, config));
+  return Status::Success;
+}
+
+Status
+AutoCompleteBackendRuntimeField(
+    RuntimeType runtime_type, inference::ModelConfig* config)
+{
+  bool fill_runtime = config->runtime().empty();
+#ifdef TRITON_ENABLE_ENSEMBLE
+  fill_runtime = fill_runtime && config->platform() != kEnsemblePlatform;
+#endif  // TRITON_ENABLE_ENSEMBLE
+  if (fill_runtime) {
+    // auto detect C++ vs Python runtime if unknown
+    if (runtime_type == RuntimeType::RUNTIME_TYPE_UNKNOWN) {
+      // default to C++ runtime
+      runtime_type = RuntimeType::RUNTIME_TYPE_CPP;
+      // unless the default model filename ends with '.py'
+      const static std::string py_model_extension = ".py";
+      const std::string& model_filename = config->default_model_filename();
+      if (model_filename.length() >= py_model_extension.length()) {
+        auto start_pos = model_filename.length() - py_model_extension.length();
+        if (model_filename.substr(start_pos) == py_model_extension) {
+          runtime_type = RuntimeType::RUNTIME_TYPE_PYTHON;
+        }
+      }
+    }
+    // set runtime library from runtime type
+    if (runtime_type == RuntimeType::RUNTIME_TYPE_CPP) {
+      if (config->backend().empty()) {
+        return Status(
+            Status::Code::INTERNAL,
+            "Model config 'backend' field cannot be empty when auto completing "
+            "for C++ 'runtime' field.");
+      }
+#ifdef _WIN32
+      config->set_runtime("triton_" + config->backend() + ".dll");
+#else
+      config->set_runtime("libtriton_" + config->backend() + ".so");
+#endif
+    } else if (runtime_type == RuntimeType::RUNTIME_TYPE_PYTHON) {
+      config->set_runtime(kPythonFilename);
+    } else {
+      return Status(Status::Code::INTERNAL, "Unimplemented runtime type.");
+    }
+  }
   return Status::Success;
 }
 
diff --git a/src/model_config_utils.h b/src/model_config_utils.h
index 018f4e7a3..36f82df49 100644
--- a/src/model_config_utils.h
+++ b/src/model_config_utils.h
@@ -42,6 +42,13 @@ enum BackendType {
   BACKEND_TYPE_PYTORCH = 4
 };
 
+/// Enumeration for different runtime types.
+enum RuntimeType {
+  RUNTIME_TYPE_UNKNOWN = 0,
+  RUNTIME_TYPE_CPP = 1,
+  RUNTIME_TYPE_PYTHON = 2
+};
+
 // Get version of a model from the path containing the model
 /// definition file.
 /// \param path The path to the model definition file.
@@ -87,9 +94,9 @@ Status GetNormalizedModelConfig(
     const std::string& model_name, const std::string& path,
     const double min_compute_capability, inference::ModelConfig* config);
 
-/// Auto-complete backend related fields (platform, backend and default model
-/// filename) if not set, note that only Triton recognized backends will be
-/// checked.
+/// Auto-complete backend related fields (platform, backend, default model
+/// filename and runtime) if not set, note that only Triton recognized backends
+/// will be checked.
 /// \param model_name The name of the model.
 /// \param model_path The full-path to the directory containing the
 /// model configuration.
@@ -99,6 +106,14 @@ Status AutoCompleteBackendFields(
     const std::string& model_name, const std::string& model_path,
     inference::ModelConfig* config);
 
+/// Auto-complete backend runtime field if not set.
+/// \param runtime_type Specify the runtime type for the model (C++ or Python).
+/// If unknown, it will be determined from the default model filename.
+/// \param config Returns the auto-completed model configuration.
+/// \return The error status.
+Status AutoCompleteBackendRuntimeField(
+    RuntimeType runtime_type, inference::ModelConfig* config);
+
 /// Detects and adds missing fields in the model configuration.
 /// \param min_compute_capability The minimum supported CUDA compute
 /// capability.

From ac7c9a1e5613f4819517ca7851f7f416cbab252b Mon Sep 17 00:00:00 2001
From: kthui <18255193+kthui@users.noreply.github.com>
Date: Wed, 1 Nov 2023 18:19:23 -0700
Subject: [PATCH 02/14] Use runtime for backend library name

---
 src/backend_config.cc     |  13 ---
 src/backend_config.h      |   4 -
 src/backend_model.cc      | 193 ++++++++++++++++++--------------------
 src/backend_model.h       |  34 +++----
 src/model_config_utils.cc |  54 +++++++----
 src/model_config_utils.h  |  11 +++
 6 files changed, 154 insertions(+), 155 deletions(-)

diff --git a/src/backend_config.cc b/src/backend_config.cc
index d69d97075..6e0b7397e 100644
--- a/src/backend_config.cc
+++ b/src/backend_config.cc
@@ -186,19 +186,6 @@ BackendConfigurationSpecializeBackendName(
   return Status::Success;
 }
 
-Status
-BackendConfigurationBackendLibraryName(
-    const std::string& backend_name, std::string* libname)
-{
-#ifdef _WIN32
-  *libname = "triton_" + backend_name + ".dll";
-#else
-  *libname = "libtriton_" + backend_name + ".so";
-#endif
-
-  return Status::Success;
-}
-
 Status
 BackendConfigurationModelLoadGpuFraction(
     const triton::common::BackendCmdlineConfigMap& config_map,
diff --git a/src/backend_config.h b/src/backend_config.h
index acd2a0c21..8e87c8be6 100644
--- a/src/backend_config.h
+++ b/src/backend_config.h
@@ -64,10 +64,6 @@ Status BackendConfigurationSpecializeBackendName(
     const triton::common::BackendCmdlineConfigMap& config_map,
     const std::string& backend_name, std::string* specialized_name);
 
-/// Return the shared library name for a backend.
-Status BackendConfigurationBackendLibraryName(
-    const std::string& backend_name, std::string* libname);
-
 /// Get GPU memory limit fraction for model loading
 /// from the backend configuration.
 Status BackendConfigurationModelLoadGpuFraction(
diff --git a/src/backend_model.cc b/src/backend_model.cc
index 87fd2cb39..1771749f9 100644
--- a/src/backend_model.cc
+++ b/src/backend_model.cc
@@ -66,15 +66,23 @@ TritonModel::Create(
 {
   model->reset();
 
-  // The model configuration must specify a backend. The name of the
-  // corresponding shared library must be libtriton_<backend>.so.
-  std::string backend_name = model_config.backend();
+  // The model configuration must specify a backend.
+  const std::string& backend_name = model_config.backend();
   if (backend_name.empty()) {
     return Status(
         Status::Code::INVALID_ARG,
         "must specify 'backend' for '" + model_config.name() + "'");
   }
 
+  // The model configuration must specify a runtime library.
+  const std::string& backend_libname = model_config.runtime();
+  if (backend_libname.empty()) {
+    return Status(
+        Status::Code::INVALID_ARG,
+        "must specify 'runtime' for '" + model_config.name() + "'");
+  }
+  bool is_python_backend_based_backend = IsPythonRuntime(backend_libname);
+
   // Localize the content of the model repository corresponding to
   // 'model_path'. This model holds a handle to the localized content
   // so that it persists as long as the model is loaded.
@@ -103,62 +111,30 @@ TritonModel::Create(
   RETURN_IF_ERROR(BackendConfigurationSpecializeBackendName(
       backend_cmdline_config_map, backend_name, &specialized_backend_name));
 
-  std::string backend_libname;
-  RETURN_IF_ERROR(BackendConfigurationBackendLibraryName(
-      specialized_backend_name, &backend_libname));
-
-  // Get the path to the backend shared library. Search path is
-  // version directory, model directory, global backend directory.
-  const auto localized_model_path = localized_model_dir->Path();
-  const auto version_path =
-      JoinPath({localized_model_path, std::to_string(version)});
-  const std::string global_path =
-      JoinPath({backend_dir, specialized_backend_name});
-  std::vector<std::string> search_paths = {
-      version_path, localized_model_path, global_path};
-
-  std::string backend_libdir;
-  std::string backend_libpath;
-  std::string python_runtime_modeldir;
+  std::vector<std::string> search_paths = GetBackendLibrarySearchPaths(
+      model_path, version, backend_dir, backend_name);
 
-  RETURN_IF_ERROR(ResolveBackendPaths(
-      specialized_backend_name, backend_dir, model_config.name(), search_paths,
-      backend_libname, &backend_libdir, &backend_libpath,
-      &python_runtime_modeldir));
-
-  // `backend_libpath` always points to shared library path.
-  if (backend_libpath.empty()) {
-    return Status(
-        Status::Code::INVALID_ARG,
-        "unable to find '" + backend_libname + "' or '" +
-            specialized_backend_name + "/" + kPythonFilename + "' for model '" +
-            model_config.name() + "', searched: " + version_path + ", " +
-            model_path + ", " + global_path);
-  }
+  std::string backend_libdir, backend_libpath;
+  RETURN_IF_ERROR(GetBackendLibraryDirectoryAndPath(
+      model_config.name(), localized_model_dir->Path(), version, backend_dir,
+      specialized_backend_name, backend_libname,
+      is_python_backend_based_backend, &search_paths, &backend_libdir,
+      &backend_libpath));
 
   // Resolve the global backend configuration with the specific backend
   // configuration
-  bool is_python_based_backend = false;
   triton::common::BackendCmdlineConfig config;
-  if (!python_runtime_modeldir.empty()) {
-    // `backend_libdir` points to model.py for python backend based backends.
-    backend_libdir = python_runtime_modeldir;
-    is_python_based_backend = true;
-    // Python backend based backends use configs, specified for python backend
-    // in cmdline.
-    RETURN_IF_ERROR(ResolveBackendConfigs(
-        backend_cmdline_config_map, kPythonBackend, config));
-  } else {
-    RETURN_IF_ERROR(ResolveBackendConfigs(
-        backend_cmdline_config_map, backend_name, config));
-  }
+  RETURN_IF_ERROR(ResolveBackendConfigs(
+      backend_cmdline_config_map,
+      (is_python_backend_based_backend ? kPythonBackend : backend_name),
+      config));
 
   RETURN_IF_ERROR(SetBackendConfigDefaults(config));
 
   std::shared_ptr<TritonBackend> backend;
   RETURN_IF_ERROR(server->BackendManager()->CreateBackend(
       backend_name, backend_libdir, backend_libpath, config,
-      is_python_based_backend, &backend));
+      is_python_backend_based_backend, &backend));
 
   // Normalize backend-dependent config
   {
@@ -317,9 +293,73 @@ TritonModel::GetExecutionPolicy(const inference::ModelConfig& model_config)
   return Status::Success;
 }
 
+std::vector<std::string>
+TritonModel::GetBackendLibrarySearchPaths(
+    const std::string& model_path, int64_t version,
+    const std::string& backend_dir, const std::string& backend_name)
+{
+  const auto version_path = JoinPath({model_path, std::to_string(version)});
+  const auto backend_path = JoinPath({backend_dir, backend_name});
+  std::vector<std::string> search_paths = {
+      version_path, model_path, backend_path};
+  return search_paths;
+}
+
 Status
-TritonModel::LocateBackendLibrary(
-    const std::vector<std::string> search_paths,
+TritonModel::GetBackendLibraryDirectoryAndPath(
+    const std::string& model_name, const std::string& model_path,
+    int64_t version, const std::string& backend_dir,
+    const std::string& backend_name, const std::string& backend_libname,
+    bool is_python_backend_based_backend,
+    std::vector<std::string>* search_paths, std::string* backend_libdir,
+    std::string* backend_libpath)
+{
+  std::string cpp_backend_libname = backend_libname;
+  if (is_python_backend_based_backend) {
+    // Use Python backend C++ runtime library.
+    cpp_backend_libname = AssembleCPPRuntimeLibraryName(kPythonBackend);
+    // The search paths only contain locations related to the Python backend
+    // based backend, the global Python backend location needs to be added.
+    search_paths->emplace_back(JoinPath({backend_dir, kPythonBackend}));
+  }
+
+  backend_libpath->clear();
+  RETURN_IF_ERROR(FindBackendLibraryPath(
+      *search_paths, cpp_backend_libname, backend_libdir, backend_libpath));
+
+  if (backend_libpath->empty()) {
+    std::string search_paths_str = "";
+    for (const auto& path : *search_paths) {
+      search_paths_str += "'" + path + "' ";
+    }
+    return Status(
+        Status::Code::INVALID_ARG,
+        "unable to find backend library '" + cpp_backend_libname +
+            "' for model '" + model_name + "', searched: " + search_paths_str);
+  }
+
+  if (is_python_backend_based_backend) {
+    // 'backend_libdir' point to Python backend based backend runtime directory.
+    *backend_libdir = JoinPath({backend_dir, backend_name});
+    // Make sure the runtime (and its directory) exists.
+    std::string runtime_libpath = JoinPath({*backend_libdir, kPythonFilename});
+    bool runtime_libpath_exist;
+    RETURN_IF_ERROR(FileExists(runtime_libpath, &runtime_libpath_exist));
+    if (!runtime_libpath_exist) {
+      return Status(
+          Status::Code::INVALID_ARG,
+          "unable to find Python backend based backend library '" +
+              backend_libname + "' for model '" + model_name +
+              "', searched: '" + runtime_libpath + "'");
+    }
+  }
+
+  return Status::Success;
+}
+
+Status
+TritonModel::FindBackendLibraryPath(
+    const std::vector<std::string>& search_paths,
     const std::string& backend_libname, std::string* backend_libdir,
     std::string* backend_libpath)
 {
@@ -337,59 +377,6 @@ TritonModel::LocateBackendLibrary(
   return Status::Success;
 }
 
-Status
-TritonModel::ResolveBackendPaths(
-    const std::string& backend_name, const std::string& global_backend_dir,
-    const std::string& model_name, std::vector<std::string>& search_paths,
-    const std::string& backend_libname, std::string* backend_libdir,
-    std::string* backend_libpath, std::string* python_runtime_modeldir)
-{
-  // Look for shared library first
-  RETURN_IF_ERROR(LocateBackendLibrary(
-      search_paths, backend_libname, backend_libdir, backend_libpath));
-
-  if (!(*backend_libpath).empty()) {
-    *python_runtime_modeldir = "";
-    return Status::Success;
-  }
-
-  // If not found, then we are processing a python-based backend.
-  // We look for libtriton_python.so in python backend directory
-  // and model.py in provided custom backend's directory
-  std::string python_backend_dir =
-      JoinPath({global_backend_dir, kPythonBackend});
-  bool is_dir;
-  RETURN_IF_ERROR(IsDirectory(python_backend_dir, &is_dir));
-  if (!is_dir) {
-    return Status(
-        Status::Code::INVALID_ARG, "unable to find '" + global_backend_dir +
-                                       "/python', '" + backend_name +
-                                       "' requires python backend to operate.");
-  }
-  search_paths.emplace_back(python_backend_dir);
-  std::string runtime_model_path =
-      JoinPath({global_backend_dir, backend_name, kPythonFilename});
-  bool exists;
-  RETURN_IF_ERROR(FileExists(runtime_model_path, &exists));
-  if (!exists) {
-    return Status(
-        Status::Code::INVALID_ARG,
-        "unable to find '" + backend_libname + "' or '" + backend_name + "/" +
-            kPythonFilename + "' for model '" + model_name + "', in " +
-            JoinPath({global_backend_dir, backend_name}));
-  }
-
-  *python_runtime_modeldir = JoinPath({global_backend_dir, backend_name});
-  std::string python_backend_libname;
-  RETURN_IF_ERROR(BackendConfigurationBackendLibraryName(
-      kPythonBackend, &python_backend_libname));
-
-  RETURN_IF_ERROR(LocateBackendLibrary(
-      search_paths, python_backend_libname, backend_libdir, backend_libpath));
-
-  return Status::Success;
-}
-
 Status
 TritonModel::ResolveBackendConfigs(
     const triton::common::BackendCmdlineConfigMap& backend_cmdline_config_map,
diff --git a/src/backend_model.h b/src/backend_model.h
index ee2be504f..544e9ee8b 100644
--- a/src/backend_model.h
+++ b/src/backend_model.h
@@ -195,25 +195,27 @@ class TritonModel : public Model {
   static Status SetBackendConfigDefaults(
       triton::common::BackendCmdlineConfig& config);
 
-  // Searches for backend_libname in provided search_paths.
-  // If found, stores backend directory in backend_libdir and
-  // backend path in backend_libpath.
-  static Status LocateBackendLibrary(
-      const std::vector<std::string> search_paths,
-      const std::string& backend_libname, std::string* backend_libdir,
+  // Get the search paths to the backend shared library.
+  static std::vector<std::string> GetBackendLibrarySearchPaths(
+      const std::string& model_path, int64_t version,
+      const std::string& backend_dir, const std::string& backend_name);
+
+  // Get backend library directory and path.
+  static Status GetBackendLibraryDirectoryAndPath(
+      const std::string& model_name, const std::string& model_path,
+      int64_t version, const std::string& backend_dir,
+      const std::string& backend_name, const std::string& backend_libname,
+      bool is_python_backend_based_backend,
+      std::vector<std::string>* search_paths, std::string* backend_libdir,
       std::string* backend_libpath);
 
-  // For a given backend (`backend_name`), looks for backend directory and
-  // location for the shared library, used by the backend. Returns:
-  // `backend_libdir` returns directory, where shared library (.so) is stored,
-  // `backend_libpath` returns the full path to .so,
-  // `python_runtime_modeldir` is set to empty string for c++ backends and
-  // returns directory, where model.py is stored.
-  static Status ResolveBackendPaths(
-      const std::string& backend_name, const std::string& global_backend_dir,
-      const std::string& model_name, std::vector<std::string>& search_paths,
+  // Search for 'backend_libname' from 'search_paths'. If found, the matching
+  // search path will be stored in 'backend_libdir' and the backend library path
+  // will be stored in 'backend_libpath'.
+  static Status FindBackendLibraryPath(
+      const std::vector<std::string>& search_paths,
       const std::string& backend_libname, std::string* backend_libdir,
-      std::string* backend_libpath, std::string* python_runtime_modeldir);
+      std::string* backend_libpath);
 
   // Clear library handles.
   void ClearHandles();
diff --git a/src/model_config_utils.cc b/src/model_config_utils.cc
index e3ae180ca..cea77d9ba 100644
--- a/src/model_config_utils.cc
+++ b/src/model_config_utils.cc
@@ -1223,31 +1223,21 @@ AutoCompleteBackendRuntimeField(
   if (fill_runtime) {
     // auto detect C++ vs Python runtime if unknown
     if (runtime_type == RuntimeType::RUNTIME_TYPE_UNKNOWN) {
-      // default to C++ runtime
-      runtime_type = RuntimeType::RUNTIME_TYPE_CPP;
-      // unless the default model filename ends with '.py'
-      const static std::string py_model_extension = ".py";
-      const std::string& model_filename = config->default_model_filename();
-      if (model_filename.length() >= py_model_extension.length()) {
-        auto start_pos = model_filename.length() - py_model_extension.length();
-        if (model_filename.substr(start_pos) == py_model_extension) {
-          runtime_type = RuntimeType::RUNTIME_TYPE_PYTHON;
-        }
+      if (IsPythonRuntime(config->default_model_filename())) {
+        runtime_type = RuntimeType::RUNTIME_TYPE_PYTHON;
+      } else {
+        runtime_type = RuntimeType::RUNTIME_TYPE_CPP;
       }
     }
     // set runtime library from runtime type
     if (runtime_type == RuntimeType::RUNTIME_TYPE_CPP) {
       if (config->backend().empty()) {
-        return Status(
-            Status::Code::INTERNAL,
-            "Model config 'backend' field cannot be empty when auto completing "
-            "for C++ 'runtime' field.");
+        LOG_INFO
+            << "Model config 'backend' field is empty when auto completing for "
+               "C++ 'runtime' field. The 'runtime' field is left unchanged.";
+      } else {
+        config->set_runtime(AssembleCPPRuntimeLibraryName(config->backend()));
       }
-#ifdef _WIN32
-      config->set_runtime("triton_" + config->backend() + ".dll");
-#else
-      config->set_runtime("libtriton_" + config->backend() + ".so");
-#endif
     } else if (runtime_type == RuntimeType::RUNTIME_TYPE_PYTHON) {
       config->set_runtime(kPythonFilename);
     } else {
@@ -2475,4 +2465,30 @@ InstanceConfigSignature(const inference::ModelInstanceGroup& instance_config)
   return config.SerializeAsString();
 }
 
+bool
+IsPythonRuntime(const std::string& library_name)
+{
+  if (library_name == kPythonFilename) {
+    return true;
+  }
+  const static std::string py_extension = ".py";
+  if (library_name.length() >= py_extension.length()) {
+    auto start_pos = library_name.length() - py_extension.length();
+    if (library_name.substr(start_pos) == py_extension) {
+      return true;
+    }
+  }
+  return false;
+}
+
+std::string
+AssembleCPPRuntimeLibraryName(const std::string& backend_name)
+{
+#ifdef _WIN32
+  return "triton_" + backend_name + ".dll";
+#else
+  return "libtriton_" + backend_name + ".so";
+#endif
+}
+
 }}  // namespace triton::core
diff --git a/src/model_config_utils.h b/src/model_config_utils.h
index 36f82df49..94f83467f 100644
--- a/src/model_config_utils.h
+++ b/src/model_config_utils.h
@@ -322,4 +322,15 @@ bool EquivalentInInstanceConfig(
 std::string InstanceConfigSignature(
     const inference::ModelInstanceGroup& instance_config);
 
+/// Check if the library name belongs to Python runtime.
+/// \param library_name The library name to check.
+/// \return True if the library name matches the 'kPythonFilename' constant or
+/// ends with a '.py' extension. False if otherwise.
+bool IsPythonRuntime(const std::string& library_name);
+
+/// Assemble the C++ runtime library name.
+/// \param backend_name The backend name.
+/// \return The assembled C++ runtime library name.
+std::string AssembleCPPRuntimeLibraryName(const std::string& backend_name);
+
 }}  // namespace triton::core

From 10edad10c729bc68796da7ad08934b265662066c Mon Sep 17 00:00:00 2001
From: kthui <18255193+kthui@users.noreply.github.com>
Date: Mon, 6 Nov 2023 22:47:37 -0800
Subject: [PATCH 03/14] Runtime may be left empty

---
 src/backend_model.cc      | 110 ++++++++++++++++++++++++++------------
 src/backend_model.h       |  23 +++++---
 src/model_config_utils.cc |  37 +++----------
 src/model_config_utils.h  |   6 ---
 4 files changed, 100 insertions(+), 76 deletions(-)

diff --git a/src/backend_model.cc b/src/backend_model.cc
index 1771749f9..03ee0e812 100644
--- a/src/backend_model.cc
+++ b/src/backend_model.cc
@@ -74,15 +74,6 @@ TritonModel::Create(
         "must specify 'backend' for '" + model_config.name() + "'");
   }
 
-  // The model configuration must specify a runtime library.
-  const std::string& backend_libname = model_config.runtime();
-  if (backend_libname.empty()) {
-    return Status(
-        Status::Code::INVALID_ARG,
-        "must specify 'runtime' for '" + model_config.name() + "'");
-  }
-  bool is_python_backend_based_backend = IsPythonRuntime(backend_libname);
-
   // Localize the content of the model repository corresponding to
   // 'model_path'. This model holds a handle to the localized content
   // so that it persists as long as the model is loaded.
@@ -111,14 +102,15 @@ TritonModel::Create(
   RETURN_IF_ERROR(BackendConfigurationSpecializeBackendName(
       backend_cmdline_config_map, backend_name, &specialized_backend_name));
 
+  bool is_python_backend_based_backend = false;
   std::vector<std::string> search_paths = GetBackendLibrarySearchPaths(
       model_path, version, backend_dir, backend_name);
 
   std::string backend_libdir, backend_libpath;
-  RETURN_IF_ERROR(GetBackendLibraryDirectoryAndPath(
+  RETURN_IF_ERROR(GetBackendLibraryProperties(
       model_config.name(), localized_model_dir->Path(), version, backend_dir,
-      specialized_backend_name, backend_libname,
-      is_python_backend_based_backend, &search_paths, &backend_libdir,
+      specialized_backend_name, model_config.runtime(),
+      &is_python_backend_based_backend, &search_paths, &backend_libdir,
       &backend_libpath));
 
   // Resolve the global backend configuration with the specific backend
@@ -306,27 +298,40 @@ TritonModel::GetBackendLibrarySearchPaths(
 }
 
 Status
-TritonModel::GetBackendLibraryDirectoryAndPath(
+TritonModel::GetBackendLibraryProperties(
     const std::string& model_name, const std::string& model_path,
     int64_t version, const std::string& backend_dir,
-    const std::string& backend_name, const std::string& backend_libname,
-    bool is_python_backend_based_backend,
+    const std::string& backend_name, const std::string& runtime,
+    bool* is_python_backend_based_backend,
     std::vector<std::string>* search_paths, std::string* backend_libdir,
     std::string* backend_libpath)
 {
+  std::string python_backend_based_backend_libdir;
+  std::string backend_libname = runtime;
+  if (backend_libname.empty()) {
+    RETURN_IF_ERROR(GetBackendRuntimeLibraryName(
+        backend_name, *search_paths, &backend_libname, backend_libdir,
+        backend_libpath, is_python_backend_based_backend));
+    if (!*is_python_backend_based_backend) {
+      // All variables are correctly set for C++ backends on initial search.
+      return Status::Success;
+    }
+    python_backend_based_backend_libdir = *backend_libdir;
+  } else {
+    *is_python_backend_based_backend = backend_libname == kPythonFilename;
+  }
+
   std::string cpp_backend_libname = backend_libname;
-  if (is_python_backend_based_backend) {
-    // Use Python backend C++ runtime library.
+  if (*is_python_backend_based_backend) {
+    // Set C++ library name to Python backend.
     cpp_backend_libname = AssembleCPPRuntimeLibraryName(kPythonBackend);
     // The search paths only contain locations related to the Python backend
-    // based backend, the global Python backend location needs to be added.
+    // based backend, the global Python backend location has to be added.
     search_paths->emplace_back(JoinPath({backend_dir, kPythonBackend}));
   }
 
-  backend_libpath->clear();
   RETURN_IF_ERROR(FindBackendLibraryPath(
       *search_paths, cpp_backend_libname, backend_libdir, backend_libpath));
-
   if (backend_libpath->empty()) {
     std::string search_paths_str = "";
     for (const auto& path : *search_paths) {
@@ -338,31 +343,70 @@ TritonModel::GetBackendLibraryDirectoryAndPath(
             "' for model '" + model_name + "', searched: " + search_paths_str);
   }
 
-  if (is_python_backend_based_backend) {
-    // 'backend_libdir' point to Python backend based backend runtime directory.
-    *backend_libdir = JoinPath({backend_dir, backend_name});
-    // Make sure the runtime (and its directory) exists.
-    std::string runtime_libpath = JoinPath({*backend_libdir, kPythonFilename});
-    bool runtime_libpath_exist;
-    RETURN_IF_ERROR(FileExists(runtime_libpath, &runtime_libpath_exist));
-    if (!runtime_libpath_exist) {
-      return Status(
-          Status::Code::INVALID_ARG,
-          "unable to find Python backend based backend library '" +
-              backend_libname + "' for model '" + model_name +
-              "', searched: '" + runtime_libpath + "'");
+  // Both 'backend_libdir' and 'backend_libpath' are now pointing to the C++
+  // backend library, 'backend_libdir' needs adjustment for Python backend based
+  // backend.
+  if (*is_python_backend_based_backend) {
+    if (python_backend_based_backend_libdir.empty()) {
+      python_backend_based_backend_libdir =
+          JoinPath({backend_dir, backend_name});
+      // Make sure the library and its directory exist.
+      std::string path =
+          JoinPath({python_backend_based_backend_libdir, kPythonFilename});
+      bool path_exist;
+      RETURN_IF_ERROR(FileExists(path, &path_exist));
+      if (!path_exist) {
+        return Status(
+            Status::Code::INVALID_ARG,
+            "unable to find Python backend based backend library '" +
+                backend_libname + "' for model '" + model_name +
+                "', searched: '" + path + "'");
+      }
     }
+    *backend_libdir = python_backend_based_backend_libdir;
   }
 
   return Status::Success;
 }
 
+Status
+TritonModel::GetBackendRuntimeLibraryName(
+    const std::string& backend_name,
+    const std::vector<std::string>& search_paths, std::string* backend_libname,
+    std::string* backend_libdir, std::string* backend_libpath,
+    bool* is_python_backend_based_backend)
+{
+  // Try C++ runtime
+  *backend_libname = AssembleCPPRuntimeLibraryName(backend_name);
+  RETURN_IF_ERROR(FindBackendLibraryPath(
+      search_paths, *backend_libname, backend_libdir, backend_libpath));
+  if (!backend_libpath->empty()) {
+    *is_python_backend_based_backend = false;
+    return Status::Success;
+  }
+  // Try Python runtime
+  *backend_libname = kPythonFilename;
+  RETURN_IF_ERROR(FindBackendLibraryPath(
+      search_paths, *backend_libname, backend_libdir, backend_libpath));
+  if (!backend_libpath->empty()) {
+    *is_python_backend_based_backend = true;
+    return Status::Success;
+  }
+  // Cannot find runtime
+  return Status(
+      Status::Code::INVALID_ARG,
+      "unable to find backend library for backend '" + backend_name +
+          "', try specifying runtime on the model configuration.");
+}
+
 Status
 TritonModel::FindBackendLibraryPath(
     const std::vector<std::string>& search_paths,
     const std::string& backend_libname, std::string* backend_libdir,
     std::string* backend_libpath)
 {
+  backend_libpath->clear();
+
   for (const auto& path : search_paths) {
     const auto full_path = JoinPath({path, backend_libname});
     bool exists = false;
diff --git a/src/backend_model.h b/src/backend_model.h
index 544e9ee8b..8495a099c 100644
--- a/src/backend_model.h
+++ b/src/backend_model.h
@@ -200,18 +200,29 @@ class TritonModel : public Model {
       const std::string& model_path, int64_t version,
       const std::string& backend_dir, const std::string& backend_name);
 
-  // Get backend library directory and path.
-  static Status GetBackendLibraryDirectoryAndPath(
+  // Get backend library directory and path, and search paths for the library
+  // and whether the backend is based on Python backend.
+  static Status GetBackendLibraryProperties(
       const std::string& model_name, const std::string& model_path,
       int64_t version, const std::string& backend_dir,
-      const std::string& backend_name, const std::string& backend_libname,
-      bool is_python_backend_based_backend,
+      const std::string& backend_name, const std::string& runtime,
+      bool* is_python_backend_based_backend,
       std::vector<std::string>* search_paths, std::string* backend_libdir,
       std::string* backend_libpath);
 
-  // Search for 'backend_libname' from 'search_paths'. If found, the matching
+  // Get 'backend_libname', 'backend_libdir', 'backend_libpath' and
+  // 'is_python_backend_based_backend' by searching for different possible
+  // backend library names on 'search_paths'
+  static Status GetBackendRuntimeLibraryName(
+      const std::string& backend_name,
+      const std::vector<std::string>& search_paths,
+      std::string* backend_libname, std::string* backend_libdir,
+      std::string* backend_libpath, bool* is_python_backend_based_backend);
+
+  // Search for 'backend_libname' on 'search_paths'. If found, the matching
   // search path will be stored in 'backend_libdir' and the backend library path
-  // will be stored in 'backend_libpath'.
+  // will be stored in 'backend_libpath'. If not found, 'backend_libpath' will
+  // be set to empty.
   static Status FindBackendLibraryPath(
       const std::vector<std::string>& search_paths,
       const std::string& backend_libname, std::string* backend_libdir,
diff --git a/src/model_config_utils.cc b/src/model_config_utils.cc
index cea77d9ba..13f9c09bb 100644
--- a/src/model_config_utils.cc
+++ b/src/model_config_utils.cc
@@ -1140,8 +1140,11 @@ AutoCompleteBackendFields(
     if (config->default_model_filename().empty()) {
       config->set_default_model_filename(kPyTorchLibTorchFilename);
     }
-    RETURN_IF_ERROR(AutoCompleteBackendRuntimeField(
-        RuntimeType::RUNTIME_TYPE_UNKNOWN, config));
+    auto runtime_type = RuntimeType::RUNTIME_TYPE_CPP;
+    if (config->default_model_filename() == kPythonFilename) {
+      runtime_type = RuntimeType::RUNTIME_TYPE_PYTHON;
+    }
+    RETURN_IF_ERROR(AutoCompleteBackendRuntimeField(runtime_type, config));
     return Status::Success;
   }
 
@@ -1202,13 +1205,9 @@ AutoCompleteBackendFields(
     config->set_backend(backend_name);
     config->set_default_model_filename(
         (std::string("model.") + backend_name).c_str());
-    RETURN_IF_ERROR(AutoCompleteBackendRuntimeField(
-        RuntimeType::RUNTIME_TYPE_UNKNOWN, config));
     return Status::Success;
   }
 
-  RETURN_IF_ERROR(AutoCompleteBackendRuntimeField(
-      RuntimeType::RUNTIME_TYPE_UNKNOWN, config));
   return Status::Success;
 }
 
@@ -1221,14 +1220,6 @@ AutoCompleteBackendRuntimeField(
   fill_runtime = fill_runtime && config->platform() != kEnsemblePlatform;
 #endif  // TRITON_ENABLE_ENSEMBLE
   if (fill_runtime) {
-    // auto detect C++ vs Python runtime if unknown
-    if (runtime_type == RuntimeType::RUNTIME_TYPE_UNKNOWN) {
-      if (IsPythonRuntime(config->default_model_filename())) {
-        runtime_type = RuntimeType::RUNTIME_TYPE_PYTHON;
-      } else {
-        runtime_type = RuntimeType::RUNTIME_TYPE_CPP;
-      }
-    }
     // set runtime library from runtime type
     if (runtime_type == RuntimeType::RUNTIME_TYPE_CPP) {
       if (config->backend().empty()) {
@@ -1241,7 +1232,7 @@ AutoCompleteBackendRuntimeField(
     } else if (runtime_type == RuntimeType::RUNTIME_TYPE_PYTHON) {
       config->set_runtime(kPythonFilename);
     } else {
-      return Status(Status::Code::INTERNAL, "Unimplemented runtime type.");
+      return Status(Status::Code::INTERNAL, "Unexpected runtime type.");
     }
   }
   return Status::Success;
@@ -2465,22 +2456,6 @@ InstanceConfigSignature(const inference::ModelInstanceGroup& instance_config)
   return config.SerializeAsString();
 }
 
-bool
-IsPythonRuntime(const std::string& library_name)
-{
-  if (library_name == kPythonFilename) {
-    return true;
-  }
-  const static std::string py_extension = ".py";
-  if (library_name.length() >= py_extension.length()) {
-    auto start_pos = library_name.length() - py_extension.length();
-    if (library_name.substr(start_pos) == py_extension) {
-      return true;
-    }
-  }
-  return false;
-}
-
 std::string
 AssembleCPPRuntimeLibraryName(const std::string& backend_name)
 {
diff --git a/src/model_config_utils.h b/src/model_config_utils.h
index 94f83467f..0b6827c87 100644
--- a/src/model_config_utils.h
+++ b/src/model_config_utils.h
@@ -322,12 +322,6 @@ bool EquivalentInInstanceConfig(
 std::string InstanceConfigSignature(
     const inference::ModelInstanceGroup& instance_config);
 
-/// Check if the library name belongs to Python runtime.
-/// \param library_name The library name to check.
-/// \return True if the library name matches the 'kPythonFilename' constant or
-/// ends with a '.py' extension. False if otherwise.
-bool IsPythonRuntime(const std::string& library_name);
-
 /// Assemble the C++ runtime library name.
 /// \param backend_name The backend name.
 /// \return The assembled C++ runtime library name.

From d9b586de64801338b91a3a4d6d6fdd094d6d31d5 Mon Sep 17 00:00:00 2001
From: kthui <18255193+kthui@users.noreply.github.com>
Date: Tue, 7 Nov 2023 16:29:42 -0800
Subject: [PATCH 04/14] Add check for well formed runtime

---
 src/backend_model.cc      |  1 +
 src/model_config_utils.cc | 15 +++++++++++++++
 src/model_config_utils.h  |  6 ++++++
 3 files changed, 22 insertions(+)

diff --git a/src/backend_model.cc b/src/backend_model.cc
index 03ee0e812..51ebc4117 100644
--- a/src/backend_model.cc
+++ b/src/backend_model.cc
@@ -318,6 +318,7 @@ TritonModel::GetBackendLibraryProperties(
     }
     python_backend_based_backend_libdir = *backend_libdir;
   } else {
+    RETURN_IF_ERROR(IsRuntimeLibraryNameWellFormed(backend_libname));
     *is_python_backend_based_backend = backend_libname == kPythonFilename;
   }
 
diff --git a/src/model_config_utils.cc b/src/model_config_utils.cc
index 13f9c09bb..df62ad03d 100644
--- a/src/model_config_utils.cc
+++ b/src/model_config_utils.cc
@@ -2466,4 +2466,19 @@ AssembleCPPRuntimeLibraryName(const std::string& backend_name)
 #endif
 }
 
+Status
+IsRuntimeLibraryNameWellFormed(const std::string& library_name)
+{
+  const static std::vector<std::string> excluded_strings = {"\\", "/"};
+  for (const auto& excluded_str : excluded_strings) {
+    if (library_name.find(excluded_str) != library_name.npos) {
+      return Status(
+          Status::Code::INVALID_ARG,
+          "Model configuration runtime field contains illegal sub-string '" +
+              excluded_str + "'");
+    }
+  }
+  return Status::Success;
+}
+
 }}  // namespace triton::core
diff --git a/src/model_config_utils.h b/src/model_config_utils.h
index 0b6827c87..0767c6bd7 100644
--- a/src/model_config_utils.h
+++ b/src/model_config_utils.h
@@ -327,4 +327,10 @@ std::string InstanceConfigSignature(
 /// \return The assembled C++ runtime library name.
 std::string AssembleCPPRuntimeLibraryName(const std::string& backend_name);
 
+/// Check if runtime library name is well formed.
+/// \param library_name The library name to check.
+/// \return Success status if '\' and '/' characters are not in library name.
+/// Error status if otherwise.
+Status IsRuntimeLibraryNameWellFormed(const std::string& library_name);
+
 }}  // namespace triton::core

From 3959a19171ad08fd449b9a6007c83010205b397f Mon Sep 17 00:00:00 2001
From: kthui <18255193+kthui@users.noreply.github.com>
Date: Thu, 9 Nov 2023 15:51:17 -0800
Subject: [PATCH 05/14] Enable conda pack support for Python backend based
 backends

---
 src/backend_model.cc      | 33 +++++++++++++++-----------
 src/backend_model.h       |  9 +++----
 src/constants.h           |  1 +
 src/model_config_utils.cc | 49 ++++++++++++++++++++++++++++++---------
 src/model_config_utils.h  |  9 +++++++
 5 files changed, 72 insertions(+), 29 deletions(-)

diff --git a/src/backend_model.cc b/src/backend_model.cc
index 51ebc4117..ebea10023 100644
--- a/src/backend_model.cc
+++ b/src/backend_model.cc
@@ -105,13 +105,17 @@ TritonModel::Create(
   bool is_python_backend_based_backend = false;
   std::vector<std::string> search_paths = GetBackendLibrarySearchPaths(
       model_path, version, backend_dir, backend_name);
-
   std::string backend_libdir, backend_libpath;
+
   RETURN_IF_ERROR(GetBackendLibraryProperties(
-      model_config.name(), localized_model_dir->Path(), version, backend_dir,
-      specialized_backend_name, model_config.runtime(),
-      &is_python_backend_based_backend, &search_paths, &backend_libdir,
-      &backend_libpath));
+      localized_model_dir->Path(), version, backend_dir,
+      specialized_backend_name, &model_config, &is_python_backend_based_backend,
+      &search_paths, &backend_libdir, &backend_libpath));
+
+  if (is_python_backend_based_backend) {
+    RETURN_IF_ERROR(SetPythonBackendBasedBackendExecutionEnvironment(
+        backend_libdir, &model_config));
+  }
 
   // Resolve the global backend configuration with the specific backend
   // configuration
@@ -299,15 +303,14 @@ TritonModel::GetBackendLibrarySearchPaths(
 
 Status
 TritonModel::GetBackendLibraryProperties(
-    const std::string& model_name, const std::string& model_path,
-    int64_t version, const std::string& backend_dir,
-    const std::string& backend_name, const std::string& runtime,
-    bool* is_python_backend_based_backend,
+    const std::string& model_path, int64_t version,
+    const std::string& backend_dir, const std::string& backend_name,
+    inference::ModelConfig* model_config, bool* is_python_backend_based_backend,
     std::vector<std::string>* search_paths, std::string* backend_libdir,
     std::string* backend_libpath)
 {
   std::string python_backend_based_backend_libdir;
-  std::string backend_libname = runtime;
+  std::string backend_libname = model_config->runtime();
   if (backend_libname.empty()) {
     RETURN_IF_ERROR(GetBackendRuntimeLibraryName(
         backend_name, *search_paths, &backend_libname, backend_libdir,
@@ -317,6 +320,7 @@ TritonModel::GetBackendLibraryProperties(
       return Status::Success;
     }
     python_backend_based_backend_libdir = *backend_libdir;
+    model_config->set_runtime(backend_libname);
   } else {
     RETURN_IF_ERROR(IsRuntimeLibraryNameWellFormed(backend_libname));
     *is_python_backend_based_backend = backend_libname == kPythonFilename;
@@ -339,9 +343,10 @@ TritonModel::GetBackendLibraryProperties(
       search_paths_str += "'" + path + "' ";
     }
     return Status(
-        Status::Code::INVALID_ARG,
-        "unable to find backend library '" + cpp_backend_libname +
-            "' for model '" + model_name + "', searched: " + search_paths_str);
+        Status::Code::INVALID_ARG, "unable to find backend library '" +
+                                       cpp_backend_libname + "' for model '" +
+                                       model_config->name() +
+                                       "', searched: " + search_paths_str);
   }
 
   // Both 'backend_libdir' and 'backend_libpath' are now pointing to the C++
@@ -360,7 +365,7 @@ TritonModel::GetBackendLibraryProperties(
         return Status(
             Status::Code::INVALID_ARG,
             "unable to find Python backend based backend library '" +
-                backend_libname + "' for model '" + model_name +
+                backend_libname + "' for model '" + model_config->name() +
                 "', searched: '" + path + "'");
       }
     }
diff --git a/src/backend_model.h b/src/backend_model.h
index 8495a099c..e475a7117 100644
--- a/src/backend_model.h
+++ b/src/backend_model.h
@@ -201,11 +201,12 @@ class TritonModel : public Model {
       const std::string& backend_dir, const std::string& backend_name);
 
   // Get backend library directory and path, and search paths for the library
-  // and whether the backend is based on Python backend.
+  // and whether the backend is based on Python backend. The model configuration
+  // runtime field will be updated if left empty.
   static Status GetBackendLibraryProperties(
-      const std::string& model_name, const std::string& model_path,
-      int64_t version, const std::string& backend_dir,
-      const std::string& backend_name, const std::string& runtime,
+      const std::string& model_path, int64_t version,
+      const std::string& backend_dir, const std::string& backend_name,
+      inference::ModelConfig* model_config,
       bool* is_python_backend_based_backend,
       std::vector<std::string>* search_paths, std::string* backend_libdir,
       std::string* backend_libpath);
diff --git a/src/constants.h b/src/constants.h
index a7c7a68cf..918a6a712 100644
--- a/src/constants.h
+++ b/src/constants.h
@@ -55,6 +55,7 @@ constexpr char kOpenVINORuntimeBackend[] = "openvino";
 
 constexpr char kPyTorchLibTorchPlatform[] = "pytorch_libtorch";
 constexpr char kPyTorchLibTorchFilename[] = "model.pt";
+constexpr char kPyTorchPythonPlatform[] = "pytorch_python";
 constexpr char kPyTorchBackend[] = "pytorch";
 
 constexpr char kPythonFilename[] = "model.py";
diff --git a/src/model_config_utils.cc b/src/model_config_utils.cc
index df62ad03d..f5d8ca95d 100644
--- a/src/model_config_utils.cc
+++ b/src/model_config_utils.cc
@@ -933,6 +933,26 @@ LocalizePythonBackendExecutionEnvironmentPath(
   return Status::Success;
 }
 
+Status
+SetPythonBackendBasedBackendExecutionEnvironment(
+    const std::string& backend_libdir, inference::ModelConfig* model_config)
+{
+  if (!model_config->parameters().contains("EXECUTION_ENV_PATH")) {
+    std::string env_name = "pb_exec_env_" + model_config->runtime() + ".tar.gz";
+    std::string env_path = JoinPath({backend_libdir, std::move(env_name)});
+    bool env_path_exist;
+    RETURN_IF_ERROR(FileExists(env_path, &env_path_exist));
+    if (env_path_exist) {
+      inference::ModelParameter model_param;
+      model_param.set_string_value(env_path);
+      (*model_config->mutable_parameters())["EXECUTION_ENV_PATH"] =
+          std::move(model_param);
+      LOG_ERROR << "\n\n" << env_path << "\n";
+    }
+  }
+  return Status::Success;
+}
+
 Status
 SetDefaultInstanceCount(
     inference::ModelInstanceGroup* group, const std::string& backend)
@@ -1134,17 +1154,23 @@ AutoCompleteBackendFields(
     }
   }
   if (config->backend() == kPyTorchBackend) {
-    if (config->platform().empty()) {
-      config->set_platform(kPyTorchLibTorchPlatform);
-    }
-    if (config->default_model_filename().empty()) {
-      config->set_default_model_filename(kPyTorchLibTorchFilename);
-    }
-    auto runtime_type = RuntimeType::RUNTIME_TYPE_CPP;
-    if (config->default_model_filename() == kPythonFilename) {
-      runtime_type = RuntimeType::RUNTIME_TYPE_PYTHON;
+    if (config->runtime() == kPythonFilename ||
+        config->default_model_filename() == kPythonFilename) {
+      if (config->platform().empty()) {
+        config->set_platform(kPyTorchPythonPlatform);
+      }
+      RETURN_IF_ERROR(AutoCompleteBackendRuntimeField(
+          RuntimeType::RUNTIME_TYPE_PYTHON, config));
+    } else {
+      if (config->platform().empty()) {
+        config->set_platform(kPyTorchLibTorchPlatform);
+      }
+      if (config->default_model_filename().empty()) {
+        config->set_default_model_filename(kPyTorchLibTorchFilename);
+      }
+      RETURN_IF_ERROR(AutoCompleteBackendRuntimeField(
+          RuntimeType::RUNTIME_TYPE_CPP, config));
     }
-    RETURN_IF_ERROR(AutoCompleteBackendRuntimeField(runtime_type, config));
     return Status::Success;
   }
 
@@ -2312,7 +2338,8 @@ GetBackendTypeFromPlatform(const std::string& platform_name)
     return BackendType::BACKEND_TYPE_ONNXRUNTIME;
   }
 
-  if (platform_name == kPyTorchLibTorchPlatform) {
+  if (platform_name == kPyTorchLibTorchPlatform ||
+      platform_name == kPyTorchPythonPlatform) {
     return BackendType::BACKEND_TYPE_PYTORCH;
   }
 
diff --git a/src/model_config_utils.h b/src/model_config_utils.h
index 0767c6bd7..7babd5737 100644
--- a/src/model_config_utils.h
+++ b/src/model_config_utils.h
@@ -144,6 +144,15 @@ Status LocalizePythonBackendExecutionEnvironmentPath(
     const std::string& model_path, inference::ModelConfig* config,
     std::shared_ptr<LocalizedPath>* localized_model_dir);
 
+/// Set execution environments for Python backend based backend, if the
+/// execution environments is not already specified and the execution
+/// environment file '<backend_libdir>/pb_exec_<runtime>.tar.gz' exists.
+/// \param backend_libdir The backend runtime library directory path.
+/// \param model_config The model configuration.
+/// \return The error status.
+Status SetPythonBackendBasedBackendExecutionEnvironment(
+    const std::string& backend_libdir, inference::ModelConfig* model_config);
+
 /// Auto-complete the instance count based on instance kind and backend name.
 /// \param group The instance group to set the count for.
 /// \param backend The backend name to check against.

From 7da30c953bf5e058b978ff52a583fb061844fe03 Mon Sep 17 00:00:00 2001
From: kthui <18255193+kthui@users.noreply.github.com>
Date: Tue, 14 Nov 2023 14:18:13 -0800
Subject: [PATCH 06/14] Rename variable name

---
 src/backend_model.cc | 29 ++++++++++++++---------------
 src/backend_model.h  |  9 ++++-----
 2 files changed, 18 insertions(+), 20 deletions(-)

diff --git a/src/backend_model.cc b/src/backend_model.cc
index ebea10023..786013686 100644
--- a/src/backend_model.cc
+++ b/src/backend_model.cc
@@ -102,17 +102,17 @@ TritonModel::Create(
   RETURN_IF_ERROR(BackendConfigurationSpecializeBackendName(
       backend_cmdline_config_map, backend_name, &specialized_backend_name));
 
-  bool is_python_backend_based_backend = false;
+  bool is_python_based_backend = false;
   std::vector<std::string> search_paths = GetBackendLibrarySearchPaths(
       model_path, version, backend_dir, backend_name);
   std::string backend_libdir, backend_libpath;
 
   RETURN_IF_ERROR(GetBackendLibraryProperties(
       localized_model_dir->Path(), version, backend_dir,
-      specialized_backend_name, &model_config, &is_python_backend_based_backend,
+      specialized_backend_name, &model_config, &is_python_based_backend,
       &search_paths, &backend_libdir, &backend_libpath));
 
-  if (is_python_backend_based_backend) {
+  if (is_python_based_backend) {
     RETURN_IF_ERROR(SetPythonBackendBasedBackendExecutionEnvironment(
         backend_libdir, &model_config));
   }
@@ -122,15 +122,14 @@ TritonModel::Create(
   triton::common::BackendCmdlineConfig config;
   RETURN_IF_ERROR(ResolveBackendConfigs(
       backend_cmdline_config_map,
-      (is_python_backend_based_backend ? kPythonBackend : backend_name),
-      config));
+      (is_python_based_backend ? kPythonBackend : backend_name), config));
 
   RETURN_IF_ERROR(SetBackendConfigDefaults(config));
 
   std::shared_ptr<TritonBackend> backend;
   RETURN_IF_ERROR(server->BackendManager()->CreateBackend(
       backend_name, backend_libdir, backend_libpath, config,
-      is_python_backend_based_backend, &backend));
+      is_python_based_backend, &backend));
 
   // Normalize backend-dependent config
   {
@@ -305,7 +304,7 @@ Status
 TritonModel::GetBackendLibraryProperties(
     const std::string& model_path, int64_t version,
     const std::string& backend_dir, const std::string& backend_name,
-    inference::ModelConfig* model_config, bool* is_python_backend_based_backend,
+    inference::ModelConfig* model_config, bool* is_python_based_backend,
     std::vector<std::string>* search_paths, std::string* backend_libdir,
     std::string* backend_libpath)
 {
@@ -314,8 +313,8 @@ TritonModel::GetBackendLibraryProperties(
   if (backend_libname.empty()) {
     RETURN_IF_ERROR(GetBackendRuntimeLibraryName(
         backend_name, *search_paths, &backend_libname, backend_libdir,
-        backend_libpath, is_python_backend_based_backend));
-    if (!*is_python_backend_based_backend) {
+        backend_libpath, is_python_based_backend));
+    if (!*is_python_based_backend) {
       // All variables are correctly set for C++ backends on initial search.
       return Status::Success;
     }
@@ -323,11 +322,11 @@ TritonModel::GetBackendLibraryProperties(
     model_config->set_runtime(backend_libname);
   } else {
     RETURN_IF_ERROR(IsRuntimeLibraryNameWellFormed(backend_libname));
-    *is_python_backend_based_backend = backend_libname == kPythonFilename;
+    *is_python_based_backend = backend_libname == kPythonFilename;
   }
 
   std::string cpp_backend_libname = backend_libname;
-  if (*is_python_backend_based_backend) {
+  if (*is_python_based_backend) {
     // Set C++ library name to Python backend.
     cpp_backend_libname = AssembleCPPRuntimeLibraryName(kPythonBackend);
     // The search paths only contain locations related to the Python backend
@@ -352,7 +351,7 @@ TritonModel::GetBackendLibraryProperties(
   // Both 'backend_libdir' and 'backend_libpath' are now pointing to the C++
   // backend library, 'backend_libdir' needs adjustment for Python backend based
   // backend.
-  if (*is_python_backend_based_backend) {
+  if (*is_python_based_backend) {
     if (python_backend_based_backend_libdir.empty()) {
       python_backend_based_backend_libdir =
           JoinPath({backend_dir, backend_name});
@@ -380,14 +379,14 @@ TritonModel::GetBackendRuntimeLibraryName(
     const std::string& backend_name,
     const std::vector<std::string>& search_paths, std::string* backend_libname,
     std::string* backend_libdir, std::string* backend_libpath,
-    bool* is_python_backend_based_backend)
+    bool* is_python_based_backend)
 {
   // Try C++ runtime
   *backend_libname = AssembleCPPRuntimeLibraryName(backend_name);
   RETURN_IF_ERROR(FindBackendLibraryPath(
       search_paths, *backend_libname, backend_libdir, backend_libpath));
   if (!backend_libpath->empty()) {
-    *is_python_backend_based_backend = false;
+    *is_python_based_backend = false;
     return Status::Success;
   }
   // Try Python runtime
@@ -395,7 +394,7 @@ TritonModel::GetBackendRuntimeLibraryName(
   RETURN_IF_ERROR(FindBackendLibraryPath(
       search_paths, *backend_libname, backend_libdir, backend_libpath));
   if (!backend_libpath->empty()) {
-    *is_python_backend_based_backend = true;
+    *is_python_based_backend = true;
     return Status::Success;
   }
   // Cannot find runtime
diff --git a/src/backend_model.h b/src/backend_model.h
index e475a7117..f6ec91f86 100644
--- a/src/backend_model.h
+++ b/src/backend_model.h
@@ -206,19 +206,18 @@ class TritonModel : public Model {
   static Status GetBackendLibraryProperties(
       const std::string& model_path, int64_t version,
       const std::string& backend_dir, const std::string& backend_name,
-      inference::ModelConfig* model_config,
-      bool* is_python_backend_based_backend,
+      inference::ModelConfig* model_config, bool* is_python_based_backend,
       std::vector<std::string>* search_paths, std::string* backend_libdir,
       std::string* backend_libpath);
 
   // Get 'backend_libname', 'backend_libdir', 'backend_libpath' and
-  // 'is_python_backend_based_backend' by searching for different possible
-  // backend library names on 'search_paths'
+  // 'is_python_based_backend' by searching for different possible backend
+  // library names on 'search_paths'
   static Status GetBackendRuntimeLibraryName(
       const std::string& backend_name,
       const std::vector<std::string>& search_paths,
       std::string* backend_libname, std::string* backend_libdir,
-      std::string* backend_libpath, bool* is_python_backend_based_backend);
+      std::string* backend_libpath, bool* is_python_based_backend);
 
   // Search for 'backend_libname' on 'search_paths'. If found, the matching
   // search path will be stored in 'backend_libdir' and the backend library path

From f22a84c452a7cf6eac8d103a393b890e8e62ac60 Mon Sep 17 00:00:00 2001
From: kthui <18255193+kthui@users.noreply.github.com>
Date: Mon, 27 Nov 2023 16:32:39 -0800
Subject: [PATCH 07/14] Shorten function name

---
 src/backend_model.cc      | 2 +-
 src/model_config_utils.cc | 2 +-
 src/model_config_utils.h  | 8 ++++----
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/backend_model.cc b/src/backend_model.cc
index 786013686..88e5fda2f 100644
--- a/src/backend_model.cc
+++ b/src/backend_model.cc
@@ -113,7 +113,7 @@ TritonModel::Create(
       &search_paths, &backend_libdir, &backend_libpath));
 
   if (is_python_based_backend) {
-    RETURN_IF_ERROR(SetPythonBackendBasedBackendExecutionEnvironment(
+    RETURN_IF_ERROR(SetPythonBasedBackendExecutionEnvironment(
         backend_libdir, &model_config));
   }
 
diff --git a/src/model_config_utils.cc b/src/model_config_utils.cc
index f5d8ca95d..b14c3e845 100644
--- a/src/model_config_utils.cc
+++ b/src/model_config_utils.cc
@@ -934,7 +934,7 @@ LocalizePythonBackendExecutionEnvironmentPath(
 }
 
 Status
-SetPythonBackendBasedBackendExecutionEnvironment(
+SetPythonBasedBackendExecutionEnvironment(
     const std::string& backend_libdir, inference::ModelConfig* model_config)
 {
   if (!model_config->parameters().contains("EXECUTION_ENV_PATH")) {
diff --git a/src/model_config_utils.h b/src/model_config_utils.h
index 7babd5737..7412a0663 100644
--- a/src/model_config_utils.h
+++ b/src/model_config_utils.h
@@ -144,13 +144,13 @@ Status LocalizePythonBackendExecutionEnvironmentPath(
     const std::string& model_path, inference::ModelConfig* config,
     std::shared_ptr<LocalizedPath>* localized_model_dir);
 
-/// Set execution environments for Python backend based backend, if the
-/// execution environments is not already specified and the execution
-/// environment file '<backend_libdir>/pb_exec_<runtime>.tar.gz' exists.
+/// Set execution environments for Python based backends, if the execution
+/// environments is not already specified and the execution environment file
+/// '<backend_libdir>/pb_exec_<runtime>.tar.gz' exists.
 /// \param backend_libdir The backend runtime library directory path.
 /// \param model_config The model configuration.
 /// \return The error status.
-Status SetPythonBackendBasedBackendExecutionEnvironment(
+Status SetPythonBasedBackendExecutionEnvironment(
     const std::string& backend_libdir, inference::ModelConfig* model_config);
 
 /// Auto-complete the instance count based on instance kind and backend name.

From 1c2c618df891abbb480eff814e7e57369a589c10 Mon Sep 17 00:00:00 2001
From: kthui <18255193+kthui@users.noreply.github.com>
Date: Mon, 27 Nov 2023 16:36:18 -0800
Subject: [PATCH 08/14] Add .. to runtime library name check

---
 src/model_config_utils.cc | 2 +-
 src/model_config_utils.h  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/model_config_utils.cc b/src/model_config_utils.cc
index b14c3e845..86323e7e4 100644
--- a/src/model_config_utils.cc
+++ b/src/model_config_utils.cc
@@ -2496,7 +2496,7 @@ AssembleCPPRuntimeLibraryName(const std::string& backend_name)
 Status
 IsRuntimeLibraryNameWellFormed(const std::string& library_name)
 {
-  const static std::vector<std::string> excluded_strings = {"\\", "/"};
+  const static std::vector<std::string> excluded_strings = {"\\", "/", ".."};
   for (const auto& excluded_str : excluded_strings) {
     if (library_name.find(excluded_str) != library_name.npos) {
       return Status(
diff --git a/src/model_config_utils.h b/src/model_config_utils.h
index 7412a0663..a6e273b3e 100644
--- a/src/model_config_utils.h
+++ b/src/model_config_utils.h
@@ -338,7 +338,7 @@ std::string AssembleCPPRuntimeLibraryName(const std::string& backend_name);
 
 /// Check if runtime library name is well formed.
 /// \param library_name The library name to check.
-/// \return Success status if '\' and '/' characters are not in library name.
+/// \return Success status if '\', '/' and '..' are not in library name.
 /// Error status if otherwise.
 Status IsRuntimeLibraryNameWellFormed(const std::string& library_name);
 

From b0e4663b320a48d5b6d9b4333dac3c9fdc33ef57 Mon Sep 17 00:00:00 2001
From: kthui <18255193+kthui@users.noreply.github.com>
Date: Mon, 27 Nov 2023 18:10:08 -0800
Subject: [PATCH 09/14] Limit Python based backend search to backend directory

---
 src/backend_model.cc | 10 ++++++----
 src/backend_model.h  |  5 +++--
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/backend_model.cc b/src/backend_model.cc
index 88e5fda2f..a05aa33f4 100644
--- a/src/backend_model.cc
+++ b/src/backend_model.cc
@@ -312,8 +312,8 @@ TritonModel::GetBackendLibraryProperties(
   std::string backend_libname = model_config->runtime();
   if (backend_libname.empty()) {
     RETURN_IF_ERROR(GetBackendRuntimeLibraryName(
-        backend_name, *search_paths, &backend_libname, backend_libdir,
-        backend_libpath, is_python_based_backend));
+        backend_dir, backend_name, *search_paths, &backend_libname,
+        backend_libdir, backend_libpath, is_python_based_backend));
     if (!*is_python_based_backend) {
       // All variables are correctly set for C++ backends on initial search.
       return Status::Success;
@@ -376,7 +376,7 @@ TritonModel::GetBackendLibraryProperties(
 
 Status
 TritonModel::GetBackendRuntimeLibraryName(
-    const std::string& backend_name,
+    const std::string& backend_dir, const std::string& backend_name,
     const std::vector<std::string>& search_paths, std::string* backend_libname,
     std::string* backend_libdir, std::string* backend_libpath,
     bool* is_python_based_backend)
@@ -390,9 +390,11 @@ TritonModel::GetBackendRuntimeLibraryName(
     return Status::Success;
   }
   // Try Python runtime
+  std::vector<std::string> python_search_paths = {
+      JoinPath({backend_dir, backend_name})};
   *backend_libname = kPythonFilename;
   RETURN_IF_ERROR(FindBackendLibraryPath(
-      search_paths, *backend_libname, backend_libdir, backend_libpath));
+      python_search_paths, *backend_libname, backend_libdir, backend_libpath));
   if (!backend_libpath->empty()) {
     *is_python_based_backend = true;
     return Status::Success;
diff --git a/src/backend_model.h b/src/backend_model.h
index f6ec91f86..486d90a42 100644
--- a/src/backend_model.h
+++ b/src/backend_model.h
@@ -212,9 +212,10 @@ class TritonModel : public Model {
 
   // Get 'backend_libname', 'backend_libdir', 'backend_libpath' and
   // 'is_python_based_backend' by searching for different possible backend
-  // library names on 'search_paths'
+  // library names on 'search_paths'. Searching for Python based backend
+  // runtime is limited to 'backend_dir'.
   static Status GetBackendRuntimeLibraryName(
-      const std::string& backend_name,
+      const std::string& backend_dir, const std::string& backend_name,
       const std::vector<std::string>& search_paths,
       std::string* backend_libname, std::string* backend_libdir,
       std::string* backend_libpath, bool* is_python_based_backend);

From 2e5b50963f4cd41b440a4359a1d038256f25179f Mon Sep 17 00:00:00 2001
From: kthui <18255193+kthui@users.noreply.github.com>
Date: Mon, 4 Dec 2023 18:20:15 -0800
Subject: [PATCH 10/14] Remove runtime autocomplete

---
 src/constants.h           |  3 --
 src/model_config_utils.cc | 78 +++++----------------------------------
 src/model_config_utils.h  | 21 ++---------
 3 files changed, 13 insertions(+), 89 deletions(-)

diff --git a/src/constants.h b/src/constants.h
index 918a6a712..da6b76706 100644
--- a/src/constants.h
+++ b/src/constants.h
@@ -55,14 +55,11 @@ constexpr char kOpenVINORuntimeBackend[] = "openvino";
 
 constexpr char kPyTorchLibTorchPlatform[] = "pytorch_libtorch";
 constexpr char kPyTorchLibTorchFilename[] = "model.pt";
-constexpr char kPyTorchPythonPlatform[] = "pytorch_python";
 constexpr char kPyTorchBackend[] = "pytorch";
 
 constexpr char kPythonFilename[] = "model.py";
 constexpr char kPythonBackend[] = "python";
 
-constexpr char kVLLMBackend[] = "vllm";
-
 #ifdef TRITON_ENABLE_ENSEMBLE
 constexpr char kEnsemblePlatform[] = "ensemble";
 #endif  // TRITON_ENABLE_ENSEMBLE
diff --git a/src/model_config_utils.cc b/src/model_config_utils.cc
index 86323e7e4..e0844271a 100644
--- a/src/model_config_utils.cc
+++ b/src/model_config_utils.cc
@@ -947,7 +947,6 @@ SetPythonBasedBackendExecutionEnvironment(
       model_param.set_string_value(env_path);
       (*model_config->mutable_parameters())["EXECUTION_ENV_PATH"] =
           std::move(model_param);
-      LOG_ERROR << "\n\n" << env_path << "\n";
     }
   }
   return Status::Success;
@@ -999,7 +998,7 @@ AutoCompleteBackendFields(
     config->set_name(model_name);
   }
 
-  // Trying to fill the 'backend', 'default_model_filename' and 'runtime' field.
+  // Trying to fill the 'backend', 'default_model_filename' field.
 
   // TensorFlow
   // For TF backend, the platform is required
@@ -1036,7 +1035,7 @@ AutoCompleteBackendFields(
     }
   }
 
-  // Fill 'backend', 'default_model_filename' and 'runtime' if missing
+  // Fill 'backend' and 'default_model_filename' if missing
   if ((config->platform() == kTensorFlowSavedModelPlatform) ||
       (config->platform() == kTensorFlowGraphDefPlatform)) {
     if (config->backend().empty()) {
@@ -1049,8 +1048,6 @@ AutoCompleteBackendFields(
         config->set_default_model_filename(kTensorFlowGraphDefFilename);
       }
     }
-    RETURN_IF_ERROR(
-        AutoCompleteBackendRuntimeField(RuntimeType::RUNTIME_TYPE_CPP, config));
     return Status::Success;
   }
 
@@ -1080,8 +1077,6 @@ AutoCompleteBackendFields(
     if (config->default_model_filename().empty()) {
       config->set_default_model_filename(kTensorRTPlanFilename);
     }
-    RETURN_IF_ERROR(
-        AutoCompleteBackendRuntimeField(RuntimeType::RUNTIME_TYPE_CPP, config));
     return Status::Success;
   }
 
@@ -1107,8 +1102,6 @@ AutoCompleteBackendFields(
     if (config->default_model_filename().empty()) {
       config->set_default_model_filename(kOnnxRuntimeOnnxFilename);
     }
-    RETURN_IF_ERROR(
-        AutoCompleteBackendRuntimeField(RuntimeType::RUNTIME_TYPE_CPP, config));
     return Status::Success;
   }
 
@@ -1129,8 +1122,6 @@ AutoCompleteBackendFields(
     if (config->default_model_filename().empty()) {
       config->set_default_model_filename(kOpenVINORuntimeOpenVINOFilename);
     }
-    RETURN_IF_ERROR(
-        AutoCompleteBackendRuntimeField(RuntimeType::RUNTIME_TYPE_CPP, config));
     return Status::Success;
   }
 
@@ -1154,22 +1145,13 @@ AutoCompleteBackendFields(
     }
   }
   if (config->backend() == kPyTorchBackend) {
-    if (config->runtime() == kPythonFilename ||
-        config->default_model_filename() == kPythonFilename) {
-      if (config->platform().empty()) {
-        config->set_platform(kPyTorchPythonPlatform);
-      }
-      RETURN_IF_ERROR(AutoCompleteBackendRuntimeField(
-          RuntimeType::RUNTIME_TYPE_PYTHON, config));
-    } else {
-      if (config->platform().empty()) {
-        config->set_platform(kPyTorchLibTorchPlatform);
-      }
-      if (config->default_model_filename().empty()) {
-        config->set_default_model_filename(kPyTorchLibTorchFilename);
-      }
-      RETURN_IF_ERROR(AutoCompleteBackendRuntimeField(
-          RuntimeType::RUNTIME_TYPE_CPP, config));
+    if (config->platform().empty()) {
+      // do not introduce new platforms, new runtimes may ignore this field.
+      config->set_platform(kPyTorchLibTorchPlatform);
+    }
+    if (config->runtime() != kPythonFilename &&
+        config->default_model_filename().empty()) {
+      config->set_default_model_filename(kPyTorchLibTorchFilename);
     }
     return Status::Success;
   }
@@ -1191,18 +1173,6 @@ AutoCompleteBackendFields(
     if (config->default_model_filename().empty()) {
       config->set_default_model_filename(kPythonFilename);
     }
-    RETURN_IF_ERROR(
-        AutoCompleteBackendRuntimeField(RuntimeType::RUNTIME_TYPE_CPP, config));
-    return Status::Success;
-  }
-
-  // vLLM
-  if (config->backend() == kVLLMBackend) {
-    if (config->default_model_filename().empty()) {
-      config->set_default_model_filename(kPythonFilename);
-    }
-    RETURN_IF_ERROR(AutoCompleteBackendRuntimeField(
-        RuntimeType::RUNTIME_TYPE_PYTHON, config));
     return Status::Success;
   }
 
@@ -1237,33 +1207,6 @@ AutoCompleteBackendFields(
   return Status::Success;
 }
 
-Status
-AutoCompleteBackendRuntimeField(
-    RuntimeType runtime_type, inference::ModelConfig* config)
-{
-  bool fill_runtime = config->runtime().empty();
-#ifdef TRITON_ENABLE_ENSEMBLE
-  fill_runtime = fill_runtime && config->platform() != kEnsemblePlatform;
-#endif  // TRITON_ENABLE_ENSEMBLE
-  if (fill_runtime) {
-    // set runtime library from runtime type
-    if (runtime_type == RuntimeType::RUNTIME_TYPE_CPP) {
-      if (config->backend().empty()) {
-        LOG_INFO
-            << "Model config 'backend' field is empty when auto completing for "
-               "C++ 'runtime' field. The 'runtime' field is left unchanged.";
-      } else {
-        config->set_runtime(AssembleCPPRuntimeLibraryName(config->backend()));
-      }
-    } else if (runtime_type == RuntimeType::RUNTIME_TYPE_PYTHON) {
-      config->set_runtime(kPythonFilename);
-    } else {
-      return Status(Status::Code::INTERNAL, "Unexpected runtime type.");
-    }
-  }
-  return Status::Success;
-}
-
 Status
 ValidateModelIOConfig(const inference::ModelConfig& config)
 {
@@ -2338,8 +2281,7 @@ GetBackendTypeFromPlatform(const std::string& platform_name)
     return BackendType::BACKEND_TYPE_ONNXRUNTIME;
   }
 
-  if (platform_name == kPyTorchLibTorchPlatform ||
-      platform_name == kPyTorchPythonPlatform) {
+  if (platform_name == kPyTorchLibTorchPlatform) {
     return BackendType::BACKEND_TYPE_PYTORCH;
   }
 
diff --git a/src/model_config_utils.h b/src/model_config_utils.h
index a6e273b3e..31e2ed723 100644
--- a/src/model_config_utils.h
+++ b/src/model_config_utils.h
@@ -42,13 +42,6 @@ enum BackendType {
   BACKEND_TYPE_PYTORCH = 4
 };
 
-/// Enumeration for different runtime types.
-enum RuntimeType {
-  RUNTIME_TYPE_UNKNOWN = 0,
-  RUNTIME_TYPE_CPP = 1,
-  RUNTIME_TYPE_PYTHON = 2
-};
-
 // Get version of a model from the path containing the model
 /// definition file.
 /// \param path The path to the model definition file.
@@ -94,9 +87,9 @@ Status GetNormalizedModelConfig(
     const std::string& model_name, const std::string& path,
     const double min_compute_capability, inference::ModelConfig* config);
 
-/// Auto-complete backend related fields (platform, backend, default model
-/// filename and runtime) if not set, note that only Triton recognized backends
-/// will be checked.
+/// Auto-complete backend related fields (platform, backend and default model
+/// filename) if not set, note that only Triton recognized backends will be
+/// checked.
 /// \param model_name The name of the model.
 /// \param model_path The full-path to the directory containing the
 /// model configuration.
@@ -106,14 +99,6 @@ Status AutoCompleteBackendFields(
     const std::string& model_name, const std::string& model_path,
     inference::ModelConfig* config);
 
-/// Auto-complete backend runtime field if not set.
-/// \param runtime_type Specify the runtime type for the model (C++ or Python).
-/// If unknown, it will be determined from the default model filename.
-/// \param config Returns the auto-completed model configuration.
-/// \return The error status.
-Status AutoCompleteBackendRuntimeField(
-    RuntimeType runtime_type, inference::ModelConfig* config);
-
 /// Detects and adds missing fields in the model configuration.
 /// \param min_compute_capability The minimum supported CUDA compute
 /// capability.

From f502bfc5f8c2c50d8e170a09b776ffd2f6d0d1f6 Mon Sep 17 00:00:00 2001
From: kthui <18255193+kthui@users.noreply.github.com>
Date: Wed, 6 Dec 2023 17:43:18 -0800
Subject: [PATCH 11/14] Update path escape check logic for runtime field

---
 src/backend_model.cc      | 34 ++++++++++++++++++++++++++--------
 src/backend_model.h       |  4 ++++
 src/filesystem/api.cc     | 15 +++++++++++++++
 src/filesystem/api.h      |  8 ++++++++
 src/model_config_utils.cc | 25 -------------------------
 src/model_config_utils.h  | 11 -----------
 src/test/CMakeLists.txt   |  4 ++++
 7 files changed, 57 insertions(+), 44 deletions(-)

diff --git a/src/backend_model.cc b/src/backend_model.cc
index f8f86ba46..5a0d4a2b7 100644
--- a/src/backend_model.cc
+++ b/src/backend_model.cc
@@ -308,7 +308,7 @@ TritonModel::GetBackendLibraryProperties(
     std::vector<std::string>* search_paths, std::string* backend_libdir,
     std::string* backend_libpath)
 {
-  std::string python_backend_based_backend_libdir;
+  std::string python_based_backend_libdir;
   std::string backend_libname = model_config->runtime();
   if (backend_libname.empty()) {
     RETURN_IF_ERROR(GetBackendRuntimeLibraryName(
@@ -318,10 +318,9 @@ TritonModel::GetBackendLibraryProperties(
       // All variables are correctly set for C++ backends on initial search.
       return Status::Success;
     }
-    python_backend_based_backend_libdir = *backend_libdir;
+    python_based_backend_libdir = *backend_libdir;
     model_config->set_runtime(backend_libname);
   } else {
-    RETURN_IF_ERROR(IsRuntimeLibraryNameWellFormed(backend_libname));
     *is_python_based_backend = backend_libname == kPythonFilename;
   }
 
@@ -347,17 +346,26 @@ TritonModel::GetBackendLibraryProperties(
                                        model_config->name() +
                                        "', searched: " + search_paths_str);
   }
+  if (IsChildPathEscapingParentPath(
+          *backend_libpath /* child_path */,
+          *backend_libdir /* parent_path */)) {
+    return Status(
+        Status::Code::INVALID_ARG,
+        "backend library name '" + cpp_backend_libname +
+            "' escapes backend directory '" + *backend_libdir +
+            "', for model '" + model_config->name() +
+            "', check model config runtime field");
+  }
 
   // Both 'backend_libdir' and 'backend_libpath' are now pointing to the C++
   // backend library, 'backend_libdir' needs adjustment for Python backend based
   // backend.
   if (*is_python_based_backend) {
-    if (python_backend_based_backend_libdir.empty()) {
-      python_backend_based_backend_libdir =
-          JoinPath({backend_dir, backend_name});
+    if (python_based_backend_libdir.empty()) {
+      python_based_backend_libdir = JoinPath({backend_dir, backend_name});
       // Make sure the library and its directory exist.
       std::string path =
-          JoinPath({python_backend_based_backend_libdir, kPythonFilename});
+          JoinPath({python_based_backend_libdir, kPythonFilename});
       bool path_exist;
       RETURN_IF_ERROR(FileExists(path, &path_exist));
       if (!path_exist) {
@@ -368,7 +376,7 @@ TritonModel::GetBackendLibraryProperties(
                 "', searched: '" + path + "'");
       }
     }
-    *backend_libdir = python_backend_based_backend_libdir;
+    *backend_libdir = python_based_backend_libdir;
   }
 
   return Status::Success;
@@ -428,6 +436,16 @@ TritonModel::FindBackendLibraryPath(
   return Status::Success;
 }
 
+std::string
+TritonModel::AssembleCPPRuntimeLibraryName(const std::string& backend_name)
+{
+#ifdef _WIN32
+  return "triton_" + backend_name + ".dll";
+#else
+  return "libtriton_" + backend_name + ".so";
+#endif
+}
+
 Status
 TritonModel::ResolveBackendConfigs(
     const triton::common::BackendCmdlineConfigMap& backend_cmdline_config_map,
diff --git a/src/backend_model.h b/src/backend_model.h
index 486d90a42..4fd66a3fa 100644
--- a/src/backend_model.h
+++ b/src/backend_model.h
@@ -229,6 +229,10 @@ class TritonModel : public Model {
       const std::string& backend_libname, std::string* backend_libdir,
       std::string* backend_libpath);
 
+  // Assemble the C++ runtime library name.
+  static std::string AssembleCPPRuntimeLibraryName(
+      const std::string& backend_name);
+
   // Clear library handles.
   void ClearHandles();
 
diff --git a/src/filesystem/api.cc b/src/filesystem/api.cc
index 2734eaf48..863827842 100644
--- a/src/filesystem/api.cc
+++ b/src/filesystem/api.cc
@@ -45,6 +45,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 
+#include <boost/filesystem.hpp>
 #include <mutex>
 
 namespace triton { namespace core {
@@ -398,6 +399,20 @@ IsAbsolutePath(const std::string& path)
   return !path.empty() && (path[0] == '/');
 }
 
+bool
+IsChildPathEscapingParentPath(
+    const std::string& child_path, const std::string& parent_path)
+{
+  // Can use std::filesystem over boost in C++17.
+  const std::string absolute_child_path =
+      boost::filesystem::weakly_canonical(child_path).string();
+  const std::string absolute_parent_path =
+      boost::filesystem::canonical(parent_path).string();
+  // Can use starts_with() over rfind() in C++20.
+  bool is_escape = absolute_child_path.rfind(absolute_parent_path, 0) != 0;
+  return is_escape;
+}
+
 std::string
 JoinPath(std::initializer_list<std::string> segments)
 {
diff --git a/src/filesystem/api.h b/src/filesystem/api.h
index 8ee0b373e..c7533ae28 100644
--- a/src/filesystem/api.h
+++ b/src/filesystem/api.h
@@ -78,6 +78,14 @@ class LocalizedPath {
 /// \return true if absolute path, false if relative path.
 bool IsAbsolutePath(const std::string& path);
 
+/// Check if the child path escapes from its parent path.
+/// \param child_path The child path.
+/// \param parent_path The parent path. The path must exist.
+/// \return true if the child path escapes from its parent path, false if the
+/// child path is within its parent path.
+bool IsChildPathEscapingParentPath(
+    const std::string& child_path, const std::string& parent_path);
+
 /// Join path segments into a longer path
 /// \param segments The path segments.
 /// \return the path formed by joining the segments.
diff --git a/src/model_config_utils.cc b/src/model_config_utils.cc
index e0844271a..72c8a36d2 100644
--- a/src/model_config_utils.cc
+++ b/src/model_config_utils.cc
@@ -2425,29 +2425,4 @@ InstanceConfigSignature(const inference::ModelInstanceGroup& instance_config)
   return config.SerializeAsString();
 }
 
-std::string
-AssembleCPPRuntimeLibraryName(const std::string& backend_name)
-{
-#ifdef _WIN32
-  return "triton_" + backend_name + ".dll";
-#else
-  return "libtriton_" + backend_name + ".so";
-#endif
-}
-
-Status
-IsRuntimeLibraryNameWellFormed(const std::string& library_name)
-{
-  const static std::vector<std::string> excluded_strings = {"\\", "/", ".."};
-  for (const auto& excluded_str : excluded_strings) {
-    if (library_name.find(excluded_str) != library_name.npos) {
-      return Status(
-          Status::Code::INVALID_ARG,
-          "Model configuration runtime field contains illegal sub-string '" +
-              excluded_str + "'");
-    }
-  }
-  return Status::Success;
-}
-
 }}  // namespace triton::core
diff --git a/src/model_config_utils.h b/src/model_config_utils.h
index 31e2ed723..e8f034a54 100644
--- a/src/model_config_utils.h
+++ b/src/model_config_utils.h
@@ -316,15 +316,4 @@ bool EquivalentInInstanceConfig(
 std::string InstanceConfigSignature(
     const inference::ModelInstanceGroup& instance_config);
 
-/// Assemble the C++ runtime library name.
-/// \param backend_name The backend name.
-/// \return The assembled C++ runtime library name.
-std::string AssembleCPPRuntimeLibraryName(const std::string& backend_name);
-
-/// Check if runtime library name is well formed.
-/// \param library_name The library name to check.
-/// \return Success status if '\', '/' and '..' are not in library name.
-/// Error status if otherwise.
-Status IsRuntimeLibraryNameWellFormed(const std::string& library_name);
-
 }}  // namespace triton::core
diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt
index 2816d52b9..eeffd99b0 100644
--- a/src/test/CMakeLists.txt
+++ b/src/test/CMakeLists.txt
@@ -126,6 +126,7 @@ if(${TRITON_ENABLE_GPU})
       ${CMAKE_CURRENT_SOURCE_DIR}/../../include
       ${GTEST_INCLUDE_DIRS}
       ${CNMEM_PATH}/include
+      ${Boost_INCLUDE_DIRS}
   )
 
   target_compile_definitions(
@@ -151,6 +152,7 @@ if(${TRITON_ENABLE_GPU})
       protobuf::libprotobuf
       ${CNMEM_LIBRARY}
       CUDA::cudart
+      Boost::filesystem
   )
 
   if (NOT WIN32)
@@ -417,6 +419,7 @@ target_include_directories(
     ${CMAKE_CURRENT_SOURCE_DIR}/..
     ${CMAKE_CURRENT_SOURCE_DIR}/../../include
     ${GTEST_INCLUDE_DIRS}
+    ${Boost_INCLUDE_DIRS}
 )
 
 target_link_libraries(
@@ -430,6 +433,7 @@ target_link_libraries(
     GTest::gtest
     GTest::gtest_main
     protobuf::libprotobuf
+    Boost::filesystem
 )
 
 install(

From 1a38e2f437c096e83a143f9370d54b9e13f3ac2a Mon Sep 17 00:00:00 2001
From: kthui <18255193+kthui@users.noreply.github.com>
Date: Mon, 8 Jan 2024 15:29:10 -0800
Subject: [PATCH 12/14] Update copyright

---
 src/backend_config.cc     | 2 +-
 src/backend_config.h      | 2 +-
 src/backend_model.cc      | 2 +-
 src/backend_model.h       | 2 +-
 src/filesystem/api.cc     | 2 +-
 src/filesystem/api.h      | 2 +-
 src/model_config_utils.cc | 2 +-
 src/model_config_utils.h  | 2 +-
 src/test/CMakeLists.txt   | 2 +-
 9 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/src/backend_config.cc b/src/backend_config.cc
index 6e0b7397e..f384aa757 100644
--- a/src/backend_config.cc
+++ b/src/backend_config.cc
@@ -1,4 +1,4 @@
-// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
diff --git a/src/backend_config.h b/src/backend_config.h
index 8e87c8be6..c55acc160 100644
--- a/src/backend_config.h
+++ b/src/backend_config.h
@@ -1,4 +1,4 @@
-// Copyright 2020-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
diff --git a/src/backend_model.cc b/src/backend_model.cc
index 5a0d4a2b7..9674a89c5 100644
--- a/src/backend_model.cc
+++ b/src/backend_model.cc
@@ -1,4 +1,4 @@
-// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
diff --git a/src/backend_model.h b/src/backend_model.h
index 4fd66a3fa..6ad1df257 100644
--- a/src/backend_model.h
+++ b/src/backend_model.h
@@ -1,4 +1,4 @@
-// Copyright 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
diff --git a/src/filesystem/api.cc b/src/filesystem/api.cc
index 863827842..b81378ada 100644
--- a/src/filesystem/api.cc
+++ b/src/filesystem/api.cc
@@ -1,4 +1,4 @@
-// Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
diff --git a/src/filesystem/api.h b/src/filesystem/api.h
index c7533ae28..67bb53e1f 100644
--- a/src/filesystem/api.h
+++ b/src/filesystem/api.h
@@ -1,4 +1,4 @@
-// Copyright (c) 2019-2020, NVIDIA CORPORATION. All rights reserved.
+// Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
diff --git a/src/model_config_utils.cc b/src/model_config_utils.cc
index 72c8a36d2..d24a3b3e1 100644
--- a/src/model_config_utils.cc
+++ b/src/model_config_utils.cc
@@ -1,4 +1,4 @@
-// Copyright 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
diff --git a/src/model_config_utils.h b/src/model_config_utils.h
index e8f034a54..8bd9af600 100644
--- a/src/model_config_utils.h
+++ b/src/model_config_utils.h
@@ -1,4 +1,4 @@
-// Copyright 2018-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// Copyright 2018-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 //
 // Redistribution and use in source and binary forms, with or without
 // modification, are permitted provided that the following conditions
diff --git a/src/test/CMakeLists.txt b/src/test/CMakeLists.txt
index eeffd99b0..a0a3177d2 100644
--- a/src/test/CMakeLists.txt
+++ b/src/test/CMakeLists.txt
@@ -1,4 +1,4 @@
-# Copyright 2019-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# Copyright 2019-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
 # modification, are permitted provided that the following conditions

From 01edc17b5ac4ee0b0fb5a73bc6b41e56b34d00e8 Mon Sep 17 00:00:00 2001
From: Jacky <18255193+kthui@users.noreply.github.com>
Date: Wed, 10 Jan 2024 15:54:39 -0800
Subject: [PATCH 13/14] Update docs wording

Co-authored-by: Ryan McCormick <rmccormick@nvidia.com>
---
 src/backend_model.cc | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/backend_model.cc b/src/backend_model.cc
index 9674a89c5..cf62e35ed 100644
--- a/src/backend_model.cc
+++ b/src/backend_model.cc
@@ -358,8 +358,7 @@ TritonModel::GetBackendLibraryProperties(
   }
 
   // Both 'backend_libdir' and 'backend_libpath' are now pointing to the C++
-  // backend library, 'backend_libdir' needs adjustment for Python backend based
-  // backend.
+  // backend library, 'backend_libdir' needs adjustment for Python based backend.
   if (*is_python_based_backend) {
     if (python_based_backend_libdir.empty()) {
       python_based_backend_libdir = JoinPath({backend_dir, backend_name});

From 34b335bd3e3d0180f0261d4fef32c70c5235be87 Mon Sep 17 00:00:00 2001
From: kthui <18255193+kthui@users.noreply.github.com>
Date: Wed, 10 Jan 2024 15:56:45 -0800
Subject: [PATCH 14/14] Fix pre-commit issue

---
 src/backend_model.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/backend_model.cc b/src/backend_model.cc
index cf62e35ed..f1175b61b 100644
--- a/src/backend_model.cc
+++ b/src/backend_model.cc
@@ -358,7 +358,8 @@ TritonModel::GetBackendLibraryProperties(
   }
 
   // Both 'backend_libdir' and 'backend_libpath' are now pointing to the C++
-  // backend library, 'backend_libdir' needs adjustment for Python based backend.
+  // backend library, 'backend_libdir' needs adjustment for Python based
+  // backend.
   if (*is_python_based_backend) {
     if (python_based_backend_libdir.empty()) {
       python_based_backend_libdir = JoinPath({backend_dir, backend_name});