diff --git a/.github/actions/build-vsix/action.yml b/.github/actions/build-vsix/action.yml index 03279fa5fbdd..16a1af28363c 100644 --- a/.github/actions/build-vsix/action.yml +++ b/.github/actions/build-vsix/action.yml @@ -21,6 +21,9 @@ runs: node-version: ${{ inputs.node_version }} cache: 'npm' + - name: Rust Tool Chain setup + uses: dtolnay/rust-toolchain@stable + # Jedi LS depends on dataclasses which is not in the stdlib in Python 3.7. - name: Use Python 3.8 for JediLSP uses: actions/setup-python@v5 @@ -44,6 +47,10 @@ runs: run: nox --session install_python_libs shell: bash + - name: Build Native Binaries + run: nox --session native_build + shell: bash + - name: Run npm ci run: npm ci --prefer-offline shell: bash diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index f7102dd5342a..ed9b10e29e2b 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -311,6 +311,30 @@ jobs: run: npm run test:functional if: matrix.test-suite == 'functional' + native-tests: + name: Native Tests + # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. + runs-on: ${{ matrix.os }} + defaults: + run: + working-directory: ${{ env.special-working-directory }} + strategy: + fail-fast: false + matrix: + # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used, + # macOS runners are expensive, and we assume that Ubuntu is enough to cover the Unix case. + os: [ubuntu-latest, windows-latest] + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + path: ${{ env.special-working-directory-relative }} + + - name: Native Locator tests + run: cargo test -- --nocapture + working-directory: ${{ env.special-working-directory }}/native_locator + smoke-tests: name: Smoke tests # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded. diff --git a/.gitignore b/.gitignore index 0556cbb2df0e..192e293bb50a 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,5 @@ dist/** *.xlf package.nls.*.json l10n/ +native_locator/target/** +native_locator/Cargo.lock diff --git a/.vscode/launch.json b/.vscode/launch.json index a4a5104c2d22..4dc107853fc6 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -252,7 +252,7 @@ ], "compounds": [ { - "name": "Debug Test Discovery", + "name": "Debug Python and Extension", "configurations": ["Python: Attach Listen", "Extension"] } ] diff --git a/.vscode/settings.json b/.vscode/settings.json index 89959f33b6b1..76501f1f6d1c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -30,6 +30,10 @@ }, "editor.defaultFormatter": "charliermarsh.ruff", }, + "[rust]": { + "editor.defaultFormatter": "rust-lang.rust-analyzer", + "editor.formatOnSave": true + }, "[typescript]": { "editor.defaultFormatter": "esbenp.prettier-vscode", "editor.formatOnSave": true @@ -67,5 +71,8 @@ "python_files/tests" ], "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true + "python.testing.pytestEnabled": true, + "rust-analyzer.linkedProjects": [ + ".\\native_locator\\Cargo.toml" + ] } diff --git a/.vscodeignore b/.vscodeignore index b96d8fcb6610..78028d097455 100644 --- a/.vscodeignore +++ b/.vscodeignore @@ -67,3 +67,8 @@ test/** tmp/** typings/** types/** +native_locator/src/** +native_locator/tests/** +native_locator/bin/** +native_locator/target/** +native_locator/Cargo.* diff --git a/native_locator/Cargo.toml b/native_locator/Cargo.toml new file mode 100644 index 000000000000..c6ae0f27c36f --- /dev/null +++ b/native_locator/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "python-finder" +version = "0.1.0" +edition = "2021" + +[target.'cfg(windows)'.dependencies] +winreg = "0.52.0" + +[dependencies] +serde = { version = "1.0.152", features = ["derive"] } +serde_json = "1.0.93" +serde_repr = "0.1.10" +regex = "1.10.4" +log = "0.4.21" +env_logger = "0.10.2" + + +[lib] +doctest = false diff --git a/native_locator/src/common_python.rs b/native_locator/src/common_python.rs new file mode 100644 index 000000000000..3d206529035c --- /dev/null +++ b/native_locator/src/common_python.rs @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +use crate::known::Environment; +use crate::locator::{Locator, LocatorResult}; +use crate::messaging::PythonEnvironment; +use crate::utils::{self, PythonEnv}; +use std::env; +use std::path::{Path, PathBuf}; + +fn get_env_path(python_executable_path: &PathBuf) -> Option { + let parent = python_executable_path.parent()?; + if parent.file_name()? == "Scripts" { + return Some(parent.parent()?.to_path_buf()); + } else { + return Some(parent.to_path_buf()); + } +} + +pub struct PythonOnPath<'a> { + pub environment: &'a dyn Environment, +} + +impl PythonOnPath<'_> { + pub fn with<'a>(environment: &'a impl Environment) -> PythonOnPath { + PythonOnPath { environment } + } +} + +impl Locator for PythonOnPath<'_> { + fn resolve(&self, env: &PythonEnv) -> Option { + let bin = if cfg!(windows) { + "python.exe" + } else { + "python" + }; + if env.executable.file_name().unwrap().to_ascii_lowercase() != bin { + return None; + } + Some(PythonEnvironment { + display_name: None, + name: None, + python_executable_path: Some(env.executable.clone()), + version: env.version.clone(), + category: crate::messaging::PythonEnvironmentCategory::System, + env_path: env.path.clone(), + env_manager: None, + project_path: None, + python_run_command: Some(vec![env.executable.to_str().unwrap().to_string()]), + }) + } + + fn find(&mut self) -> Option { + let paths = self.environment.get_env_var("PATH".to_string())?; + let bin = if cfg!(windows) { + "python.exe" + } else { + "python" + }; + + // Exclude files from this folder, as they would have been discovered elsewhere (widows_store) + // Also the exe is merely a pointer to another file. + let home = self.environment.get_user_home()?; + let apps_path = Path::new(&home) + .join("AppData") + .join("Local") + .join("Microsoft") + .join("WindowsApps"); + let mut environments: Vec = vec![]; + env::split_paths(&paths) + .filter(|p| !p.starts_with(apps_path.clone())) + .map(|p| p.join(bin)) + .filter(|p| p.exists()) + .for_each(|full_path| { + let version = utils::get_version(&full_path); + let env_path = get_env_path(&full_path); + if let Some(env) = self.resolve(&PythonEnv::new(full_path, env_path, version)) { + environments.push(env); + } + }); + + if environments.is_empty() { + None + } else { + Some(LocatorResult { + environments, + managers: vec![], + }) + } + } +} diff --git a/native_locator/src/conda.rs b/native_locator/src/conda.rs new file mode 100644 index 000000000000..929a991689cc --- /dev/null +++ b/native_locator/src/conda.rs @@ -0,0 +1,1046 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +use crate::known; +use crate::known::Environment; +use crate::locator::Locator; +use crate::locator::LocatorResult; +use crate::messaging; +use crate::messaging::EnvManager; +use crate::messaging::EnvManagerType; +use crate::messaging::PythonEnvironment; +use crate::utils::PythonEnv; +use crate::utils::{find_python_binary_path, get_environment_key, get_environment_manager_key}; +use log::trace; +use log::warn; +use regex::Regex; +use std::collections::HashMap; +use std::collections::HashSet; +use std::env; +use std::path::{Path, PathBuf}; + +/// Specifically returns the file names that are valid for 'conda' on windows +/// Path is relative to the installation folder of conda. +#[cfg(windows)] +fn get_relative_paths_to_conda_executable() -> Vec { + vec![ + PathBuf::from("Scripts").join("conda.exe"), + PathBuf::from("Scripts").join("conda.bat"), + ] +} + +/// Specifically returns the file names that are valid for 'conda' on linux/Mac +/// Path is relative to the installation folder of conda. +#[cfg(unix)] +fn get_relative_paths_to_conda_executable() -> Vec { + vec![PathBuf::from("bin").join("conda")] +} + +/// Returns the relative path to the python executable for the conda installation. +/// Path is relative to the installation folder of conda. +/// In windows the python.exe for the conda installation is in the root folder. +#[cfg(windows)] +fn get_relative_paths_to_main_python_executable() -> PathBuf { + PathBuf::from("python.exe") +} + +/// Returns the relative path to the python executable for the conda installation. +/// Path is relative to the installation folder of conda. +/// In windows the python.exe for the conda installation is in the bin folder. +#[cfg(unix)] +fn get_relative_paths_to_main_python_executable() -> PathBuf { + PathBuf::from("bin").join("python") +} + +#[derive(Debug)] +struct CondaPackage { + #[allow(dead_code)] + path: PathBuf, + version: String, +} + +/// Get the path to the json file along with the version of a package in the conda environment from the 'conda-meta' directory. +fn get_conda_package_json_path(path: &Path, package: &str) -> Option { + // conda-meta is in the root of the conda installation folder + let path = path.join("conda-meta"); + let package_name = format!("{}-", package); + let regex = Regex::new(format!("^{}-((\\d+\\.*)*)-.*.json$", package).as_str()); + std::fs::read_dir(path) + .ok()? + .filter_map(Result::ok) + .find_map(|entry| { + let path = entry.path(); + let file_name = path.file_name()?.to_string_lossy(); + if file_name.starts_with(&package_name) && file_name.ends_with(".json") { + match regex.clone().ok().unwrap().captures(&file_name)?.get(1) { + Some(version) => Some(CondaPackage { + path: path.clone(), + version: version.as_str().to_string(), + }), + None => None, + } + } else { + None + } + }) +} + +fn get_conda_executable(path: &PathBuf) -> Option { + for relative_path in get_relative_paths_to_conda_executable() { + let exe = path.join(&relative_path); + if exe.exists() { + return Some(exe); + } + } + + None +} + +/// Specifically returns the file names that are valid for 'conda' on windows +#[cfg(windows)] +fn get_conda_bin_names() -> Vec<&'static str> { + vec!["conda.exe", "conda.bat"] +} + +/// Specifically returns the file names that are valid for 'conda' on linux/Mac +#[cfg(unix)] +fn get_conda_bin_names() -> Vec<&'static str> { + vec!["conda"] +} + +/// Find the conda binary on the PATH environment variable +fn find_conda_binary_on_path(environment: &dyn known::Environment) -> Option { + let paths = environment.get_env_var("PATH".to_string())?; + for path in env::split_paths(&paths) { + for bin in get_conda_bin_names() { + let conda_path = path.join(bin); + if let Ok(metadata) = std::fs::metadata(&conda_path) { + if metadata.is_file() || metadata.is_symlink() { + return Some(conda_path); + } + } + } + } + None +} + +#[cfg(windows)] +fn get_known_conda_locations(environment: &dyn known::Environment) -> Vec { + let user_profile = environment.get_env_var("USERPROFILE".to_string()).unwrap(); + let program_data = environment.get_env_var("PROGRAMDATA".to_string()).unwrap(); + let all_user_profile = environment + .get_env_var("ALLUSERSPROFILE".to_string()) + .unwrap(); + let home_drive = environment.get_env_var("HOMEDRIVE".to_string()).unwrap(); + let mut known_paths = vec![ + Path::new(&user_profile).join("Anaconda3\\Scripts"), + Path::new(&program_data).join("Anaconda3\\Scripts"), + Path::new(&all_user_profile).join("Anaconda3\\Scripts"), + Path::new(&home_drive).join("Anaconda3\\Scripts"), + Path::new(&user_profile).join("Miniconda3\\Scripts"), + Path::new(&program_data).join("Miniconda3\\Scripts"), + Path::new(&all_user_profile).join("Miniconda3\\Scripts"), + Path::new(&home_drive).join("Miniconda3\\Scripts"), + ]; + known_paths.append(&mut environment.get_know_global_search_locations()); + known_paths +} + +#[cfg(unix)] +fn get_known_conda_locations(environment: &dyn known::Environment) -> Vec { + let mut known_paths = vec![ + PathBuf::from("/opt/anaconda3/bin"), + PathBuf::from("/opt/miniconda3/bin"), + PathBuf::from("/usr/local/anaconda3/bin"), + PathBuf::from("/usr/local/miniconda3/bin"), + PathBuf::from("/usr/anaconda3/bin"), + PathBuf::from("/usr/miniconda3/bin"), + PathBuf::from("/home/anaconda3/bin"), + PathBuf::from("/home/miniconda3/bin"), + PathBuf::from("/anaconda3/bin"), + PathBuf::from("/miniconda3/bin"), + ]; + if let Some(home) = environment.get_user_home() { + known_paths.push(PathBuf::from(home.clone()).join("anaconda3/bin")); + known_paths.push(PathBuf::from(home).join("miniconda3/bin")); + } + known_paths.append(&mut environment.get_know_global_search_locations()); + known_paths +} + +/// Find conda binary in known locations +fn find_conda_binary_in_known_locations(environment: &dyn known::Environment) -> Option { + let conda_bin_names = get_conda_bin_names(); + let known_locations = get_known_conda_locations(environment); + for location in known_locations { + for bin in &conda_bin_names { + let conda_path = location.join(bin); + if let Some(metadata) = std::fs::metadata(&conda_path).ok() { + if metadata.is_file() || metadata.is_symlink() { + return Some(conda_path); + } + } + } + } + None +} + +/// Find the conda binary on the system +pub fn find_conda_binary(environment: &dyn known::Environment) -> Option { + let conda_binary_on_path = find_conda_binary_on_path(environment); + match conda_binary_on_path { + Some(conda_binary_on_path) => Some(conda_binary_on_path), + None => find_conda_binary_in_known_locations(environment), + } +} + +fn get_conda_manager(path: &PathBuf) -> Option { + let conda_exe = get_conda_executable(path)?; + let conda_pkg = get_conda_package_json_path(path, "conda")?; + + Some(EnvManager { + executable_path: conda_exe, + version: Some(conda_pkg.version), + tool: EnvManagerType::Conda, + }) +} + +#[derive(Debug, Clone)] +struct CondaEnvironment { + name: String, + named: bool, + env_path: PathBuf, + python_executable_path: Option, + version: Option, + conda_install_folder: Option, +} +fn get_conda_environment_info(env_path: &PathBuf, named: bool) -> Option { + let metadata = env_path.metadata(); + if let Ok(metadata) = metadata { + if metadata.is_dir() { + let conda_install_folder = get_conda_installation_used_to_create_conda_env(env_path); + let env_path = env_path.clone(); + if let Some(python_binary) = find_python_binary_path(&env_path) { + if let Some(package_info) = get_conda_package_json_path(&env_path, "python") { + return Some(CondaEnvironment { + name: env_path.file_name()?.to_string_lossy().to_string(), + env_path, + named, + python_executable_path: Some(python_binary), + version: Some(package_info.version), + conda_install_folder, + }); + } else { + return Some(CondaEnvironment { + name: env_path.file_name()?.to_string_lossy().to_string(), + env_path, + named, + python_executable_path: Some(python_binary), + version: None, + conda_install_folder, + }); + } + } else { + return Some(CondaEnvironment { + name: env_path.file_name()?.to_string_lossy().to_string(), + env_path, + named, + python_executable_path: None, + version: None, + conda_install_folder, + }); + } + } + } + + None +} + +fn get_environments_from_envs_folder_in_conda_directory( + path: &Path, +) -> Option> { + let mut envs: Vec = vec![]; + // iterate through all sub directories in the env folder + // for each sub directory, check if it has a python executable + // if it does, create a PythonEnvironment object and add it to the list + for entry in std::fs::read_dir(path.join("envs")) + .ok()? + .filter_map(Result::ok) + { + if let Some(env) = get_conda_environment_info(&entry.path(), true) { + envs.push(env); + } + } + + Some(envs) +} + +fn get_conda_envs_from_environment_txt(environment: &dyn known::Environment) -> Vec { + let mut envs = vec![]; + if let Some(home) = environment.get_user_home() { + let home = Path::new(&home); + let environment_txt = home.join(".conda").join("environments.txt"); + if let Ok(reader) = std::fs::read_to_string(environment_txt.clone()) { + trace!("Found environments.txt file {:?}", environment_txt); + for line in reader.lines() { + envs.push(line.to_string()); + } + } + } + envs +} + +#[derive(Debug)] +struct Condarc { + env_dirs: Vec, +} + +/** + * Get the list of conda environments found in other locations such as + * /.conda/envs + * /AppData/Local/conda/conda/envs + */ +pub fn get_conda_environment_paths_from_conda_rc( + environment: &dyn known::Environment, +) -> Vec { + if let Some(paths) = get_conda_conda_rc(environment) { + paths.env_dirs + } else { + vec![] + } +} + +fn get_conda_environment_paths_from_known_paths( + environment: &dyn known::Environment, +) -> Vec { + if let Some(home) = environment.get_user_home() { + let mut env_paths: Vec = vec![]; + let _ = [ + PathBuf::from(".conda").join("envs"), + PathBuf::from("AppData") + .join("Local") + .join("conda") + .join("conda") + .join("envs"), + ] + .iter() + .map(|path| { + let full_path = home.join(path); + for entry in std::fs::read_dir(full_path).ok()?.filter_map(Result::ok) { + if entry.path().is_dir() { + trace!("Search for conda envs in location {:?}", entry.path()); + env_paths.push(entry.path()); + } + } + None::<()> + }); + return env_paths; + } + vec![] +} + +#[cfg(windows)] +fn get_conda_rc_search_paths(environment: &dyn known::Environment) -> Vec { + let mut search_paths: Vec = vec![ + "C:\\ProgramData\\conda\\.condarc", + "C:\\ProgramData\\conda\\condarc", + "C:\\ProgramData\\conda\\condarc.d", + ] + .iter() + .map(|p| PathBuf::from(p)) + .collect(); + + if let Some(conda_root) = environment.get_env_var("CONDA_ROOT".to_string()) { + search_paths.append(&mut vec![ + PathBuf::from(conda_root.clone()).join(".condarc"), + PathBuf::from(conda_root.clone()).join("condarc"), + PathBuf::from(conda_root.clone()).join(".condarc.d"), + ]); + } + if let Some(home) = environment.get_user_home() { + search_paths.append(&mut vec![ + home.join(".config").join("conda").join(".condarc"), + home.join(".config").join("conda").join("condarc"), + home.join(".config").join("conda").join("condarc.d"), + home.join(".conda").join(".condarc"), + home.join(".conda").join("condarc"), + home.join(".conda").join("condarc.d"), + home.join(".condarc"), + ]); + } + if let Some(conda_prefix) = environment.get_env_var("CONDA_PREFIX".to_string()) { + search_paths.append(&mut vec![ + PathBuf::from(conda_prefix.clone()).join(".condarc"), + PathBuf::from(conda_prefix.clone()).join("condarc"), + PathBuf::from(conda_prefix.clone()).join(".condarc.d"), + ]); + } + if let Some(condarc) = environment.get_env_var("CONDARC".to_string()) { + search_paths.append(&mut vec![PathBuf::from(condarc)]); + } + + search_paths +} +#[cfg(unix)] +fn get_conda_rc_search_paths(environment: &dyn known::Environment) -> Vec { + let mut search_paths: Vec = vec![ + "/etc/conda/.condarc", + "/etc/conda/condarc", + "/etc/conda/condarc.d/", + "/var/lib/conda/.condarc", + "/var/lib/conda/condarc", + "/var/lib/conda/condarc.d/", + ] + .iter() + .map(|p| PathBuf::from(p)) + .map(|p| { + // This only applies in tests. + // We need this, as the root folder cannot be mocked. + if let Some(root) = environment.get_root() { + root.join(p.to_string_lossy()[1..].to_string()) + } else { + p + } + }) + .collect(); + + if let Some(conda_root) = environment.get_env_var("CONDA_ROOT".to_string()) { + search_paths.append(&mut vec![ + PathBuf::from(conda_root.clone()).join(".condarc"), + PathBuf::from(conda_root.clone()).join("condarc"), + PathBuf::from(conda_root.clone()).join(".condarc.d"), + ]); + } + if let Some(xdg_config_home) = environment.get_env_var("XDG_CONFIG_HOME".to_string()) { + search_paths.append(&mut vec![ + PathBuf::from(xdg_config_home.clone()).join(".condarc"), + PathBuf::from(xdg_config_home.clone()).join("condarc"), + PathBuf::from(xdg_config_home.clone()).join(".condarc.d"), + ]); + } + if let Some(home) = environment.get_user_home() { + search_paths.append(&mut vec![ + home.join(".config").join("conda").join(".condarc"), + home.join(".config").join("conda").join("condarc"), + home.join(".config").join("conda").join("condarc.d"), + home.join(".conda").join(".condarc"), + home.join(".conda").join("condarc"), + home.join(".conda").join("condarc.d"), + home.join(".condarc"), + ]); + } + if let Some(conda_prefix) = environment.get_env_var("CONDA_PREFIX".to_string()) { + search_paths.append(&mut vec![ + PathBuf::from(conda_prefix.clone()).join(".condarc"), + PathBuf::from(conda_prefix.clone()).join("condarc"), + PathBuf::from(conda_prefix.clone()).join(".condarc.d"), + ]); + } + if let Some(condarc) = environment.get_env_var("CONDARC".to_string()) { + search_paths.append(&mut vec![PathBuf::from(condarc)]); + } + + search_paths +} + +/** + * The .condarc file contains a list of directories where conda environments are created. + * https://conda.io/projects/conda/en/latest/configuration.html#envs-dirs + * + * TODO: Search for the .condarc file in the following locations: + * https://conda.io/projects/conda/en/latest/user-guide/configuration/use-condarc.html#searching-for-condarc + */ +fn get_conda_conda_rc(environment: &dyn known::Environment) -> Option { + let conda_rc = get_conda_rc_search_paths(environment) + .into_iter() + .find(|p| p.exists())?; + let mut start_consuming_values = false; + trace!("conda_rc: {:?}", conda_rc); + let reader = std::fs::read_to_string(conda_rc).ok()?; + let mut env_dirs = vec![]; + for line in reader.lines() { + if line.starts_with("envs_dirs:") && !start_consuming_values { + start_consuming_values = true; + continue; + } + if start_consuming_values { + if line.trim().starts_with("-") { + if let Some(env_dir) = line.splitn(2, '-').nth(1) { + let env_dir = PathBuf::from(env_dir.trim()).join("envs"); + if env_dir.exists() { + env_dirs.push(env_dir); + } + } + continue; + } else { + break; + } + } + } + return Some(Condarc { env_dirs }); +} + +/** + * When we create conda environments in specific folder using the -p argument, the location of the conda executable is not know. + * If the user has multiple conda installations, any one of those could have created that specific environment. + * Fortunately the conda-meta/history file contains the path to the conda executable (script) that was used to create the environment. + * The format of the file is as follows: + * # cmd: C:\Users\user\miniconda3\Scripts\conda-script.py create --name myenv + * + * Thus all we need to do is to look for the 'cmd' line in the file and extract the path to the conda executable and match that against the path provided. + */ +fn was_conda_environment_created_by_specific_conda( + env: &CondaEnvironment, + root_conda_path: &PathBuf, +) -> bool { + if let Some(cmd_line) = env.conda_install_folder.clone() { + if cmd_line + .to_lowercase() + .contains(&root_conda_path.to_string_lossy().to_lowercase()) + { + return true; + } else { + return false; + } + } + + false +} + +/** + * The conda-meta/history file in conda environments contain the command used to create the conda environment. + * And example is `# cmd: \Scripts\conda-script.py create -n sample`` + * And example is `# cmd: conda create -n sample`` + * + * Sometimes the cmd line contains the fully qualified path to the conda install folder. + * This function returns the path to the conda installation that was used to create the environment. + */ +fn get_conda_installation_used_to_create_conda_env(env_path: &PathBuf) -> Option { + let conda_meta_history = env_path.join("conda-meta").join("history"); + if let Ok(reader) = std::fs::read_to_string(conda_meta_history.clone()) { + if let Some(line) = reader.lines().map(|l| l.trim()).find(|l| { + l.to_lowercase().starts_with("# cmd:") && l.to_lowercase().contains(" create -") + }) { + // Sample lines + // # cmd: \Scripts\conda-script.py create -n samlpe1 + // # cmd: \Scripts\conda-script.py create -p + // # cmd: /Users/donjayamanne/miniconda3/bin/conda create -n conda1 + let start_index = line.to_lowercase().find("# cmd:")? + "# cmd:".len(); + let end_index = line.to_lowercase().find(" create -")?; + let cmd_line = PathBuf::from(line[start_index..end_index].trim().to_string()); + if let Some(cmd_line) = cmd_line.parent() { + if let Some(name) = cmd_line.file_name() { + if name.to_ascii_lowercase() == "bin" || name.to_ascii_lowercase() == "scripts" + { + if let Some(cmd_line) = cmd_line.parent() { + return Some(cmd_line.to_str()?.to_string()); + } + } + return Some(cmd_line.to_str()?.to_string()); + } + } + } + } + + None +} + +#[cfg(windows)] +fn get_known_conda_install_locations(environment: &dyn known::Environment) -> Vec { + let user_profile = environment.get_env_var("USERPROFILE".to_string()).unwrap(); + let program_data = environment.get_env_var("PROGRAMDATA".to_string()).unwrap(); + let all_user_profile = environment + .get_env_var("ALLUSERSPROFILE".to_string()) + .unwrap(); + let home_drive = environment.get_env_var("HOMEDRIVE".to_string()).unwrap(); + let mut known_paths = vec![ + Path::new(&user_profile).join("Anaconda3"), + Path::new(&program_data).join("Anaconda3"), + Path::new(&all_user_profile).join("Anaconda3"), + Path::new(&home_drive).join("Anaconda3"), + Path::new(&user_profile).join("Miniconda3"), + Path::new(&program_data).join("Miniconda3"), + Path::new(&all_user_profile).join("Miniconda3"), + Path::new(&home_drive).join("Miniconda3"), + Path::new(&all_user_profile).join("miniforge3"), + Path::new(&home_drive).join("miniforge3"), + ]; + if let Some(home) = environment.get_user_home() { + known_paths.push(PathBuf::from(home.clone()).join("anaconda3")); + known_paths.push(PathBuf::from(home.clone()).join("miniconda3")); + known_paths.push(PathBuf::from(home.clone()).join("miniforge3")); + known_paths.push(PathBuf::from(home).join(".conda")); + } + known_paths +} + +#[cfg(unix)] +fn get_known_conda_install_locations(environment: &dyn known::Environment) -> Vec { + let mut known_paths = vec![ + PathBuf::from("/opt/anaconda3"), + PathBuf::from("/opt/miniconda3"), + PathBuf::from("/usr/local/anaconda3"), + PathBuf::from("/usr/local/miniconda3"), + PathBuf::from("/usr/anaconda3"), + PathBuf::from("/usr/miniconda3"), + PathBuf::from("/home/anaconda3"), + PathBuf::from("/home/miniconda3"), + PathBuf::from("/anaconda3"), + PathBuf::from("/miniconda3"), + PathBuf::from("/miniforge3"), + PathBuf::from("/miniforge3"), + ]; + if let Some(home) = environment.get_user_home() { + known_paths.push(PathBuf::from(home.clone()).join("anaconda3")); + known_paths.push(PathBuf::from(home.clone()).join("miniconda3")); + known_paths.push(PathBuf::from(home.clone()).join("miniforge3")); + known_paths.push(PathBuf::from(home).join(".conda")); + } + known_paths +} + +fn get_activation_command(env: &CondaEnvironment, manager: &EnvManager) -> Option> { + if env.python_executable_path.is_none() { + return None; + } + let conda_exe = manager.executable_path.to_str().unwrap().to_string(); + if env.named { + Some(vec![ + conda_exe, + "run".to_string(), + "-n".to_string(), + env.name.clone(), + "python".to_string(), + ]) + } else { + Some(vec![ + conda_exe, + "run".to_string(), + "-p".to_string(), + env.env_path.to_str().unwrap().to_string(), + "python".to_string(), + ]) + } +} + +fn get_root_python_environment(path: &PathBuf, manager: &EnvManager) -> Option { + let python_exe = path.join(get_relative_paths_to_main_python_executable()); + if !python_exe.exists() { + return None; + } + if let Some(package_info) = get_conda_package_json_path(&path, "python") { + let conda_exe = manager.executable_path.to_str().unwrap().to_string(); + return Some(PythonEnvironment { + display_name: None, + name: None, + category: messaging::PythonEnvironmentCategory::Conda, + python_executable_path: Some(python_exe), + version: Some(package_info.version), + env_path: Some(path.clone()), + env_manager: Some(manager.clone()), + python_run_command: Some(vec![ + conda_exe, + "run".to_string(), + "-p".to_string(), + path.to_str().unwrap().to_string(), + "python".to_string(), + ]), + project_path: None, + }); + } + None +} + +fn get_conda_environments_in_specified_install_path( + conda_install_folder: &PathBuf, + possible_conda_envs: &mut HashMap, +) -> Option { + let mut managers: Vec = vec![]; + let mut environments: Vec = vec![]; + let mut detected_envs: HashSet = HashSet::new(); + let mut detected_managers: HashSet = HashSet::new(); + if conda_install_folder.is_dir() && conda_install_folder.exists() { + if let Some(manager) = get_conda_manager(&conda_install_folder) { + // 1. Base environment. + if let Some(env) = get_root_python_environment(&conda_install_folder, &manager) { + if let Some(env_path) = env.clone().env_path { + possible_conda_envs.remove(&env_path); + let key = env_path.to_string_lossy().to_string(); + if !detected_envs.contains(&key) { + detected_envs.insert(key); + environments.push(env); + } + } + } + + // 2. All environments in the `/envs` folder + let mut envs: Vec = vec![]; + if let Some(environments) = + get_environments_from_envs_folder_in_conda_directory(conda_install_folder) + { + environments.iter().for_each(|env| { + possible_conda_envs.remove(&env.env_path); + envs.push(env.clone()); + }); + } + + // 3. All environments in the environments.txt and other locations (such as `conda config --show envs_dirs`) + // Only include those environments that were created by the specific conda installation + // Ignore environments that are in the env sub directory of the conda folder, as those would have been + // tracked elsewhere, we're only interested in conda envs located in other parts of the file system created using the -p flag. + // E.g conda_install_folder is `/` + // Then all folders such as `//envs/env1` can be ignored + // As these would have been discovered in previous step. + for (key, env) in possible_conda_envs.clone().iter() { + if env + .env_path + .to_string_lossy() + .contains(conda_install_folder.to_str().unwrap()) + { + continue; + } + if was_conda_environment_created_by_specific_conda(&env, conda_install_folder) { + envs.push(env.clone()); + possible_conda_envs.remove(key); + } + } + + // Finally construct the PythonEnvironment objects + envs.iter().for_each(|env| { + let exe = env.python_executable_path.clone(); + let env = PythonEnvironment::new( + None, + Some(env.name.clone()), + exe.clone(), + messaging::PythonEnvironmentCategory::Conda, + env.version.clone(), + Some(env.env_path.clone()), + Some(manager.clone()), + get_activation_command(env, &manager), + ); + if let Some(key) = get_environment_key(&env) { + if !detected_envs.contains(&key) { + detected_envs.insert(key); + environments.push(env); + } + } + }); + + let key = get_environment_manager_key(&manager); + if !detected_managers.contains(&key) { + detected_managers.insert(key); + managers.push(manager); + } + } + } + + if managers.is_empty() && environments.is_empty() { + return None; + } + + Some(LocatorResult { + managers, + environments, + }) +} + +fn find_conda_environments_from_known_conda_install_locations( + environment: &dyn known::Environment, + possible_conda_envs: &mut HashMap, +) -> Option { + let mut managers: Vec = vec![]; + let mut environments: Vec = vec![]; + + // We know conda is installed in `/Anaconda3`, `/miniforge3`, etc + // Look for these and discover all environments in these locations + for possible_conda_install_folder in get_known_conda_install_locations(environment) { + if let Some(mut result) = get_conda_environments_in_specified_install_path( + &possible_conda_install_folder, + possible_conda_envs, + ) { + managers.append(&mut result.managers); + environments.append(&mut result.environments); + } + } + + // We know conda environments are listed in the `environments.txt` file + // Sometimes the base environment is also listed in these paths + // Go through them an look for possible conda install folders in these paths. + // & then look for conda environments in each of them. + // This accounts for cases where Conda install location is in some un-common (custom) location + let mut env_paths_to_remove: Vec = vec![]; + for (key, env) in possible_conda_envs + .clone() + .iter() + .filter(|(_, env)| is_conda_install_location(&env.env_path)) + { + if let Some(mut result) = + get_conda_environments_in_specified_install_path(&env.env_path, possible_conda_envs) + { + possible_conda_envs.remove(key); + managers.append(&mut result.managers); + environments.append(&mut result.environments); + env_paths_to_remove.push(env.env_path.clone()); + } + } + + if managers.is_empty() && environments.is_empty() { + return None; + } + + Some(LocatorResult { + managers, + environments, + }) +} + +fn is_conda_install_location(path: &PathBuf) -> bool { + let envs_path = path.join("envs"); + return envs_path.exists() && envs_path.is_dir(); +} + +pub fn get_conda_version(conda_binary: &PathBuf) -> Option { + let mut parent = conda_binary.parent()?; + if parent.ends_with("bin") { + parent = parent.parent()?; + } + if parent.ends_with("Library") { + parent = parent.parent()?; + } + match get_conda_package_json_path(&parent, "conda") { + Some(result) => Some(result.version), + None => match get_conda_package_json_path(&parent.parent()?, "conda") { + Some(result) => Some(result.version), + None => None, + }, + } +} + +fn get_known_conda_envs_from_various_locations( + environment: &dyn known::Environment, +) -> HashMap { + let mut env_paths = get_conda_envs_from_environment_txt(environment) + .iter() + .map(|e| PathBuf::from(e)) + .collect::>(); + + let mut env_paths_from_conda_rc = get_conda_environment_paths_from_conda_rc(environment); + env_paths.append(&mut env_paths_from_conda_rc); + + let mut envs_from_known_paths = get_conda_environment_paths_from_known_paths(environment); + env_paths.append(&mut envs_from_known_paths); + + let mut envs: Vec = vec![]; + env_paths.iter().for_each(|path| { + if !path.exists() { + return; + } + if let Some(env) = get_conda_environment_info(&path, false) { + envs.push(env); + } + }); + + envs.into_iter().fold(HashMap::new(), |mut acc, env| { + acc.insert(env.env_path.clone(), env); + acc + }) +} + +fn get_conda_environments_from_known_locations_that_have_not_been_discovered( + known_environment: &Vec, + environment: &dyn known::Environment, + undiscovered_environments: &mut HashMap, +) -> Option { + if undiscovered_environments.is_empty() { + return None; + } + + // Ok, weird, we have an environment in environments.txt file that was not discovered. + // Let's try to discover it. + warn!( + "Found environments in environments.txt that were not discovered: {:?}", + undiscovered_environments + ); + + let manager = match known_environment + .iter() + .find_map(|env| env.env_manager.as_ref()) + { + Some(manager) => Some(manager.clone()), + None => { + // Old approach of finding the conda executable. + let conda_binary = find_conda_binary(environment)?; + Some(EnvManager::new( + conda_binary.clone(), + get_conda_version(&conda_binary), + EnvManagerType::Conda, + )) + } + }; + + if let Some(manager) = manager { + let mut environments: Vec = vec![]; + for (_, env) in undiscovered_environments { + let exe = env.python_executable_path.clone(); + let env = PythonEnvironment::new( + None, + Some(env.name.clone()), + exe.clone(), + messaging::PythonEnvironmentCategory::Conda, + env.version.clone(), + Some(env.env_path.clone()), + Some(manager.clone()), + get_activation_command(&env, &manager), + ); + environments.push(env); + } + if environments.len() > 0 { + return Some(LocatorResult { + managers: vec![manager], + environments, + }); + } + } else { + warn!("Could not find conda executable to discover environments in environments.txt"); + } + + None +} + +pub struct Conda<'a> { + pub manager: Option, + pub environment: &'a dyn Environment, + pub discovered_environment_paths: HashSet, + pub discovered_managers: HashSet, +} + +pub trait CondaLocator { + fn find_in(&mut self, possible_conda_folder: &PathBuf) -> Option; +} + +impl Conda<'_> { + pub fn with<'a>(environment: &'a impl Environment) -> Conda { + Conda { + environment, + manager: None, + discovered_environment_paths: HashSet::new(), + discovered_managers: HashSet::new(), + } + } + fn filter_result(&mut self, result: Option) -> Option { + if let Some(result) = result { + let envs: Vec = result + .environments + .iter() + .filter(|e| { + if let Some(env_path) = e.env_path.clone() { + if self.discovered_environment_paths.contains(&env_path) { + return false; + } + self.discovered_environment_paths.insert(env_path); + return true; + } + false + }) + .cloned() + .collect(); + + let managers: Vec = result + .managers + .iter() + .filter(|e| { + let key = get_environment_manager_key(e); + if self.discovered_managers.contains(&key) { + return false; + } + self.discovered_managers.insert(key); + return true; + }) + .cloned() + .collect(); + + if envs.len() > 0 || managers.len() > 0 { + return Some(LocatorResult { + managers: managers, + environments: envs, + }); + } + } + None + } +} + +impl CondaLocator for Conda<'_> { + fn find_in(&mut self, possible_conda_folder: &PathBuf) -> Option { + let mut possible_conda_envs = get_known_conda_envs_from_various_locations(self.environment); + self.filter_result(get_conda_environments_in_specified_install_path( + possible_conda_folder, + &mut possible_conda_envs, + )) + } +} + +impl Locator for Conda<'_> { + fn resolve(&self, _env: &PythonEnv) -> Option { + // We will find everything in find + None + } + + fn find(&mut self) -> Option { + let mut managers: Vec = vec![]; + let mut environments: Vec = vec![]; + let mut detected_managers: HashSet = HashSet::new(); + let mut possible_conda_envs = get_known_conda_envs_from_various_locations(self.environment); + + if let Some(result) = + self.filter_result(find_conda_environments_from_known_conda_install_locations( + self.environment, + &mut possible_conda_envs, + )) + { + result.managers.iter().for_each(|m| { + detected_managers.insert(get_environment_manager_key(m)); + managers.push(m.clone()); + }); + + result + .environments + .iter() + .for_each(|e| environments.push(e.clone())); + } + + if let Some(result) = self.filter_result( + get_conda_environments_from_known_locations_that_have_not_been_discovered( + &environments, + self.environment, + &mut possible_conda_envs, + ), + ) { + result.managers.iter().for_each(|m| { + let key = get_environment_manager_key(m); + if !detected_managers.contains(&key) { + warn!("Found a new manager using the fallback mechanism: {:?}", m); + detected_managers.insert(key); + managers.push(m.clone()); + } + }); + + result.environments.iter().for_each(|e| { + warn!( + "Found a new conda environment using the fallback mechanism: {:?}", + e + ); + environments.push(e.clone()); + }); + } + + if managers.is_empty() && environments.is_empty() { + return None; + } + + Some(LocatorResult { + managers, + environments, + }) + } +} diff --git a/native_locator/src/global_virtualenvs.rs b/native_locator/src/global_virtualenvs.rs new file mode 100644 index 000000000000..8004775e3ee2 --- /dev/null +++ b/native_locator/src/global_virtualenvs.rs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + known, + utils::{find_python_binary_path, get_version, PythonEnv}, +}; +use std::{fs, path::PathBuf}; + +pub fn get_global_virtualenv_dirs(environment: &impl known::Environment) -> Vec { + let mut venv_dirs: Vec = vec![]; + + if let Some(work_on_home) = environment.get_env_var("WORKON_HOME".to_string()) { + if let Ok(work_on_home) = fs::canonicalize(work_on_home) { + if work_on_home.exists() { + venv_dirs.push(work_on_home); + } + } + } + + if let Some(home) = environment.get_user_home() { + let home = PathBuf::from(home); + for dir in [ + PathBuf::from("envs"), + PathBuf::from(".direnv"), + PathBuf::from(".venvs"), + PathBuf::from(".virtualenvs"), + PathBuf::from(".local").join("share").join("virtualenvs"), + ] { + let venv_dir = home.join(dir); + if venv_dir.exists() { + venv_dirs.push(venv_dir); + } + } + if cfg!(target_os = "linux") { + let envs = PathBuf::from("Envs"); + if envs.exists() { + venv_dirs.push(envs); + } + } + } + + venv_dirs +} + +pub fn list_global_virtual_envs(environment: &impl known::Environment) -> Vec { + let mut python_envs: Vec = vec![]; + for root_dir in get_global_virtualenv_dirs(environment).iter() { + if let Ok(dirs) = fs::read_dir(root_dir) { + for venv_dir in dirs { + if let Ok(venv_dir) = venv_dir { + let venv_dir = venv_dir.path(); + if !venv_dir.is_dir() { + continue; + } + if let Some(executable) = find_python_binary_path(&venv_dir) { + python_envs.push(PythonEnv::new( + executable.clone(), + Some(venv_dir), + get_version(&executable), + )); + } + } + } + } + } + + python_envs +} diff --git a/native_locator/src/homebrew.rs b/native_locator/src/homebrew.rs new file mode 100644 index 000000000000..f51e783e12c8 --- /dev/null +++ b/native_locator/src/homebrew.rs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + known::Environment, + locator::{Locator, LocatorResult}, + messaging::PythonEnvironment, + utils::PythonEnv, +}; +use regex::Regex; +use std::{collections::HashSet, fs::DirEntry, path::PathBuf}; + +fn is_symlinked_python_executable(path: DirEntry) -> Option { + let path = path.path(); + let name = path.file_name()?.to_string_lossy(); + if !name.starts_with("python") || name.ends_with("-config") || name.ends_with("-build") { + return None; + } + let metadata = std::fs::symlink_metadata(&path).ok()?; + if metadata.is_file() || !metadata.file_type().is_symlink() { + return None; + } + Some(std::fs::canonicalize(path).ok()?) +} + +pub struct Homebrew<'a> { + pub environment: &'a dyn Environment, +} + +impl Homebrew<'_> { + pub fn with<'a>(environment: &'a impl Environment) -> Homebrew { + Homebrew { environment } + } +} + +impl Locator for Homebrew<'_> { + fn resolve(&self, _env: &PythonEnv) -> Option { + None + } + + fn find(&mut self) -> Option { + let homebrew_prefix = self + .environment + .get_env_var("HOMEBREW_PREFIX".to_string())?; + let homebrew_prefix_bin = PathBuf::from(homebrew_prefix).join("bin"); + let mut reported: HashSet = HashSet::new(); + let python_regex = Regex::new(r"/(\d+\.\d+\.\d+)/").unwrap(); + let mut environments: Vec = vec![]; + for file in std::fs::read_dir(homebrew_prefix_bin) + .ok()? + .filter_map(Result::ok) + { + if let Some(exe) = is_symlinked_python_executable(file) { + let python_version = exe.to_string_lossy().to_string(); + let version = match python_regex.captures(&python_version) { + Some(captures) => match captures.get(1) { + Some(version) => Some(version.as_str().to_string()), + None => None, + }, + None => None, + }; + if reported.contains(&exe.to_string_lossy().to_string()) { + continue; + } + reported.insert(exe.to_string_lossy().to_string()); + let env = crate::messaging::PythonEnvironment::new( + None, + None, + Some(exe.clone()), + crate::messaging::PythonEnvironmentCategory::Homebrew, + version, + None, + None, + Some(vec![exe.to_string_lossy().to_string()]), + ); + environments.push(env); + } + } + if environments.is_empty() { + None + } else { + Some(LocatorResult { + managers: vec![], + environments, + }) + } + } +} diff --git a/native_locator/src/known.rs b/native_locator/src/known.rs new file mode 100644 index 000000000000..6e37d897157e --- /dev/null +++ b/native_locator/src/known.rs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +use std::{env, path::PathBuf}; + +pub trait Environment { + fn get_user_home(&self) -> Option; + /** + * Only used in tests, this is the root `/`. + */ + fn get_root(&self) -> Option; + fn get_env_var(&self, key: String) -> Option; + fn get_know_global_search_locations(&self) -> Vec; +} + +pub struct EnvironmentApi {} + +#[cfg(windows)] +impl Environment for EnvironmentApi { + fn get_user_home(&self) -> Option { + get_user_home() + } + fn get_root(&self) -> Option { + None + } + fn get_env_var(&self, key: String) -> Option { + get_env_var(key) + } + fn get_know_global_search_locations(&self) -> Vec { + vec![] + } +} + +#[cfg(unix)] +impl Environment for EnvironmentApi { + fn get_user_home(&self) -> Option { + get_user_home() + } + fn get_root(&self) -> Option { + None + } + fn get_env_var(&self, key: String) -> Option { + get_env_var(key) + } + fn get_know_global_search_locations(&self) -> Vec { + vec![ + PathBuf::from("/usr/bin"), + PathBuf::from("/usr/local/bin"), + PathBuf::from("/bin"), + PathBuf::from("/home/bin"), + PathBuf::from("/sbin"), + PathBuf::from("/usr/sbin"), + PathBuf::from("/usr/local/sbin"), + PathBuf::from("/home/sbin"), + PathBuf::from("/opt"), + PathBuf::from("/opt/bin"), + PathBuf::from("/opt/sbin"), + PathBuf::from("/opt/homebrew/bin"), + ] + } +} + +fn get_user_home() -> Option { + let home = env::var("HOME").or_else(|_| env::var("USERPROFILE")); + match home { + Ok(home) => Some(PathBuf::from(home)), + Err(_) => None, + } +} + +fn get_env_var(key: String) -> Option { + match env::var(key) { + Ok(path) => Some(path), + Err(_) => None, + } +} diff --git a/native_locator/src/lib.rs b/native_locator/src/lib.rs new file mode 100644 index 000000000000..ba353c71ce12 --- /dev/null +++ b/native_locator/src/lib.rs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +pub mod messaging; +pub mod utils; +pub mod common_python; +pub mod logging; +pub mod conda; +pub mod known; +pub mod pyenv; +pub mod global_virtualenvs; +pub mod virtualenvwrapper; +pub mod pipenv; +pub mod virtualenv; +pub mod venv; +pub mod locator; +pub mod windows_registry; +pub mod windows_store; diff --git a/native_locator/src/locator.rs b/native_locator/src/locator.rs new file mode 100644 index 000000000000..a318c102230a --- /dev/null +++ b/native_locator/src/locator.rs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + messaging::{EnvManager, PythonEnvironment}, + utils::PythonEnv, +}; + +#[derive(Debug, Clone)] +pub struct LocatorResult { + pub managers: Vec, + pub environments: Vec, +} + +pub trait Locator { + /** + * Given a Python environment, this will convert it to a PythonEnvironment that can be supported by this locator. + * If an environment is not supported by this locator, this will return None. + * + * I.e. use this to test whether an environment is of a specific type. + */ + fn resolve(&self, env: &PythonEnv) -> Option; + /** + * Finds all environments specific to this locator. + */ + fn find(&mut self) -> Option; +} diff --git a/native_locator/src/logging.rs b/native_locator/src/logging.rs new file mode 100644 index 000000000000..66532ff67eff --- /dev/null +++ b/native_locator/src/logging.rs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +use serde::{Deserialize, Serialize}; + +#[derive(Serialize, Deserialize, PartialEq, Debug, Eq, Clone)] +pub enum LogLevel { + #[serde(rename = "debug")] + Debug, + #[serde(rename = "info")] + Info, + #[serde(rename = "warning")] + Warning, + #[serde(rename = "error")] + Error, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Log { + pub message: String, + pub level: LogLevel, +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct LogMessage { + pub jsonrpc: String, + pub method: String, + pub params: Log, +} + +impl LogMessage { + pub fn new(message: String, level: LogLevel) -> Self { + Self { + jsonrpc: "2.0".to_string(), + method: "log".to_string(), + params: Log { message, level }, + } + } +} diff --git a/native_locator/src/main.rs b/native_locator/src/main.rs new file mode 100644 index 000000000000..ee976bf756d2 --- /dev/null +++ b/native_locator/src/main.rs @@ -0,0 +1,115 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +use crate::messaging::initialize_logger; +use global_virtualenvs::list_global_virtual_envs; +use known::EnvironmentApi; +use locator::Locator; +use log::LevelFilter; +use messaging::{create_dispatcher, JsonRpcDispatcher, MessageDispatcher}; +use std::time::SystemTime; +use utils::PythonEnv; + +mod common_python; +mod conda; +mod global_virtualenvs; +mod homebrew; +mod known; +mod locator; +mod logging; +mod messaging; +mod pipenv; +mod pyenv; +mod utils; +mod venv; +mod virtualenv; +mod virtualenvwrapper; +mod windows_registry; +mod windows_store; + +fn main() { + let environment = EnvironmentApi {}; + initialize_logger(LevelFilter::Debug); + + log::info!("Starting Native Locator"); + let now = SystemTime::now(); + let mut dispatcher = create_dispatcher(); + + let virtualenv_locator = virtualenv::VirtualEnv::new(); + let venv_locator = venv::Venv::new(); + let virtualenvwrapper_locator = virtualenvwrapper::VirtualEnvWrapper::with(&environment); + let pipenv_locator = pipenv::PipEnv::new(); + let mut path_locator = common_python::PythonOnPath::with(&environment); + let mut conda_locator = conda::Conda::with(&environment); + + #[cfg(unix)] + let mut homebrew_locator = homebrew::Homebrew::with(&environment); + #[cfg(windows)] + let mut windows_store = windows_store::WindowsStore::with(&environment); + #[cfg(windows)] + let mut windows_registry = windows_registry::WindowsRegistry::with(&mut conda_locator); + + // Step 1: These environments take precedence over all others. + // As they are very specific and guaranteed to be specific type. + #[cfg(windows)] + find_environments(&mut windows_registry, &mut dispatcher); + let mut pyenv_locator = pyenv::PyEnv::with(&environment, &mut conda_locator); + find_environments(&mut pyenv_locator, &mut dispatcher); + #[cfg(unix)] + find_environments(&mut homebrew_locator, &mut dispatcher); + find_environments(&mut conda_locator, &mut dispatcher); + #[cfg(windows)] + find_environments(&mut windows_store, &mut dispatcher); + + // Step 2: Search in some global locations. + for env in list_global_virtual_envs(&environment).iter() { + if dispatcher.was_environment_reported(&env) { + continue; + } + + let _ = resolve_environment(&pipenv_locator, env, &mut dispatcher) + || resolve_environment(&virtualenvwrapper_locator, env, &mut dispatcher) + || resolve_environment(&venv_locator, env, &mut dispatcher) + || resolve_environment(&virtualenv_locator, env, &mut dispatcher); + } + + // Step 3: Finally find in the current PATH variable + find_environments(&mut path_locator, &mut dispatcher); + + match now.elapsed() { + Ok(elapsed) => { + log::info!("Native Locator took {} milliseconds.", elapsed.as_millis()); + } + Err(e) => { + log::error!("Error getting elapsed time: {:?}", e); + } + } + + dispatcher.exit(); +} + +fn resolve_environment( + locator: &dyn Locator, + env: &PythonEnv, + dispatcher: &mut JsonRpcDispatcher, +) -> bool { + if let Some(env) = locator.resolve(env) { + dispatcher.report_environment(env); + return true; + } + false +} + +fn find_environments(locator: &mut dyn Locator, dispatcher: &mut JsonRpcDispatcher) -> Option<()> { + if let Some(result) = locator.find() { + result + .environments + .iter() + .for_each(|e| dispatcher.report_environment(e.clone())); + result + .managers + .iter() + .for_each(|m| dispatcher.report_environment_manager(m.clone())); + } + Some(()) +} diff --git a/native_locator/src/messaging.rs b/native_locator/src/messaging.rs new file mode 100644 index 000000000000..73e708dcac5f --- /dev/null +++ b/native_locator/src/messaging.rs @@ -0,0 +1,234 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +use crate::{ + logging::{LogLevel, LogMessage}, + utils::{get_environment_key, get_environment_manager_key, PythonEnv}, +}; +use env_logger::Builder; +use log::LevelFilter; +use serde::{Deserialize, Serialize}; +use std::{collections::HashSet, path::PathBuf}; + +pub trait MessageDispatcher { + fn was_environment_reported(&self, env: &PythonEnv) -> bool; + fn report_environment_manager(&mut self, env: EnvManager) -> (); + fn report_environment(&mut self, env: PythonEnvironment) -> (); + fn exit(&mut self) -> (); +} + +#[derive(Serialize, Deserialize, Copy, Clone)] +#[serde(rename_all = "camelCase")] +#[derive(Debug)] +pub enum EnvManagerType { + Conda, + Pyenv, +} + +#[derive(Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +#[derive(Debug)] +pub struct EnvManager { + pub executable_path: PathBuf, + pub version: Option, + pub tool: EnvManagerType, +} + +impl EnvManager { + pub fn new(executable_path: PathBuf, version: Option, tool: EnvManagerType) -> Self { + Self { + executable_path, + version, + tool, + } + } +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[derive(Debug)] +pub struct EnvManagerMessage { + pub jsonrpc: String, + pub method: String, + pub params: EnvManager, +} + +impl EnvManagerMessage { + pub fn new(params: EnvManager) -> Self { + Self { + jsonrpc: "2.0".to_string(), + method: "envManager".to_string(), + params, + } + } +} + +#[derive(Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +#[derive(Debug)] +pub enum PythonEnvironmentCategory { + System, + Homebrew, + Conda, + Pyenv, + PyenvVirtualEnv, + WindowsStore, + WindowsRegistry, + Pipenv, + VirtualEnvWrapper, + Venv, + VirtualEnv, +} + +#[derive(Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +#[derive(Debug)] +pub struct PythonEnvironment { + pub display_name: Option, + pub name: Option, + pub python_executable_path: Option, + pub category: PythonEnvironmentCategory, + pub version: Option, + pub env_path: Option, + pub env_manager: Option, + pub python_run_command: Option>, + /** + * The project path for the Pipenv environment. + */ + pub project_path: Option, +} + +impl PythonEnvironment { + pub fn new( + display_name: Option, + name: Option, + python_executable_path: Option, + category: PythonEnvironmentCategory, + version: Option, + env_path: Option, + env_manager: Option, + python_run_command: Option>, + ) -> Self { + Self { + display_name, + name, + python_executable_path, + category, + version, + env_path, + env_manager, + python_run_command, + project_path: None, + } + } +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[derive(Debug)] +pub struct PythonEnvironmentMessage { + pub jsonrpc: String, + pub method: String, + pub params: PythonEnvironment, +} + +impl PythonEnvironmentMessage { + pub fn new(params: PythonEnvironment) -> Self { + Self { + jsonrpc: "2.0".to_string(), + method: "pythonEnvironment".to_string(), + params, + } + } +} + +#[derive(Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +#[derive(Debug)] +pub struct ExitMessage { + pub jsonrpc: String, + pub method: String, + pub params: Option<()>, +} + +impl ExitMessage { + pub fn new() -> Self { + Self { + jsonrpc: "2.0".to_string(), + method: "exit".to_string(), + params: None, + } + } +} + +pub struct JsonRpcDispatcher { + pub reported_managers: HashSet, + pub reported_environments: HashSet, +} +pub fn send_message(message: T) -> () { + let message = serde_json::to_string(&message).unwrap(); + print!( + "Content-Length: {}\r\nContent-Type: application/vscode-jsonrpc; charset=utf-8\r\n\r\n{}", + message.len(), + message + ); +} + +pub fn initialize_logger(log_level: LevelFilter) { + Builder::new() + .format(|_, record| { + let level = match record.level() { + log::Level::Debug => LogLevel::Debug, + log::Level::Error => LogLevel::Error, + log::Level::Info => LogLevel::Info, + log::Level::Warn => LogLevel::Warning, + _ => LogLevel::Debug, + }; + send_message(LogMessage::new( + format!("{}", record.args()).to_string(), + level, + )); + Ok(()) + }) + .filter(None, log_level) + .init(); +} + +impl JsonRpcDispatcher {} +impl MessageDispatcher for JsonRpcDispatcher { + fn was_environment_reported(&self, env: &PythonEnv) -> bool { + if let Some(key) = env.executable.as_os_str().to_str() { + return self.reported_environments.contains(key); + } + false + } + + fn report_environment_manager(&mut self, env: EnvManager) -> () { + let key = get_environment_manager_key(&env); + if !self.reported_managers.contains(&key) { + self.reported_managers.insert(key); + send_message(EnvManagerMessage::new(env)); + } + } + fn report_environment(&mut self, env: PythonEnvironment) -> () { + if let Some(key) = get_environment_key(&env) { + if !self.reported_environments.contains(&key) { + self.reported_environments.insert(key); + send_message(PythonEnvironmentMessage::new(env.clone())); + } + if let Some(manager) = env.env_manager { + self.report_environment_manager(manager); + } + } + } + fn exit(&mut self) -> () { + send_message(ExitMessage::new()); + } +} + +pub fn create_dispatcher() -> JsonRpcDispatcher { + JsonRpcDispatcher { + reported_managers: HashSet::new(), + reported_environments: HashSet::new(), + } +} diff --git a/native_locator/src/pipenv.rs b/native_locator/src/pipenv.rs new file mode 100644 index 000000000000..bb5eab5776fe --- /dev/null +++ b/native_locator/src/pipenv.rs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::locator::{Locator, LocatorResult}; +use crate::messaging::PythonEnvironment; +use crate::utils::PythonEnv; +use std::fs; +use std::path::PathBuf; + +fn get_pipenv_project(env: &PythonEnv) -> Option { + let project_file = env.path.clone()?.join(".project"); + if let Ok(contents) = fs::read_to_string(project_file) { + let project_folder = PathBuf::from(contents.trim().to_string()); + if project_folder.exists() { + return Some(project_folder); + } + } + None +} + +pub struct PipEnv {} + +impl PipEnv { + pub fn new() -> PipEnv { + PipEnv {} + } +} + +impl Locator for PipEnv { + fn resolve(&self, env: &PythonEnv) -> Option { + let project_path = get_pipenv_project(env)?; + Some(PythonEnvironment { + display_name: None, + name: None, + python_executable_path: Some(env.executable.clone()), + category: crate::messaging::PythonEnvironmentCategory::Pipenv, + version: env.version.clone(), + env_path: env.path.clone(), + env_manager: None, + python_run_command: Some(vec![env.executable.to_str().unwrap().to_string()]), + project_path: Some(project_path), + }) + } + + fn find(&mut self) -> Option { + None + } +} diff --git a/native_locator/src/pyenv.rs b/native_locator/src/pyenv.rs new file mode 100644 index 000000000000..9137a80df1f7 --- /dev/null +++ b/native_locator/src/pyenv.rs @@ -0,0 +1,232 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::conda::CondaLocator; +use crate::known; +use crate::known::Environment; +use crate::locator::Locator; +use crate::locator::LocatorResult; +use crate::messaging; +use crate::messaging::EnvManager; +use crate::messaging::EnvManagerType; +use crate::messaging::PythonEnvironment; +use crate::utils::find_and_parse_pyvenv_cfg; +use crate::utils::find_python_binary_path; +use crate::utils::PythonEnv; +use regex::Regex; +use std::fs; +use std::path::PathBuf; + +#[cfg(windows)] +fn get_home_pyenv_dir(environment: &dyn known::Environment) -> Option { + let home = environment.get_user_home()?; + Some(PathBuf::from(home).join(".pyenv").join("pyenv-win")) +} + +#[cfg(unix)] +fn get_home_pyenv_dir(environment: &dyn known::Environment) -> Option { + let home = environment.get_user_home()?; + Some(PathBuf::from(home).join(".pyenv")) +} + +fn get_binary_from_known_paths(environment: &dyn known::Environment) -> Option { + for known_path in environment.get_know_global_search_locations() { + let bin = known_path.join("pyenv"); + if bin.exists() { + return Some(bin); + } + } + None +} + +fn get_pyenv_dir(environment: &dyn known::Environment) -> Option { + // Check if the pyenv environment variables exist: PYENV on Windows, PYENV_ROOT on Unix. + // They contain the path to pyenv's installation folder. + // If they don't exist, use the default path: ~/.pyenv/pyenv-win on Windows, ~/.pyenv on Unix. + // If the interpreter path starts with the path to the pyenv folder, then it is a pyenv environment. + // See https://github.com/pyenv/pyenv#locating-the-python-installation for general usage, + // And https://github.com/pyenv-win/pyenv-win for Windows specifics. + + match environment.get_env_var("PYENV_ROOT".to_string()) { + Some(dir) => Some(PathBuf::from(dir)), + None => match environment.get_env_var("PYENV".to_string()) { + Some(dir) => Some(PathBuf::from(dir)), + None => get_home_pyenv_dir(environment), + }, + } +} + +fn get_pyenv_binary(environment: &dyn known::Environment) -> Option { + let dir = get_pyenv_dir(environment)?; + let exe = PathBuf::from(dir).join("bin").join("pyenv"); + if fs::metadata(&exe).is_ok() { + Some(exe) + } else { + get_binary_from_known_paths(environment) + } +} + +fn get_pyenv_version(folder_name: &String) -> Option { + // Stable Versions = like 3.10.10 + let python_regex = Regex::new(r"^(\d+\.\d+\.\d+)$").unwrap(); + match python_regex.captures(&folder_name) { + Some(captures) => match captures.get(1) { + Some(version) => Some(version.as_str().to_string()), + None => None, + }, + None => { + // Dev Versions = like 3.10-dev + let python_regex = Regex::new(r"^(\d+\.\d+-dev)$").unwrap(); + match python_regex.captures(&folder_name) { + Some(captures) => match captures.get(1) { + Some(version) => Some(version.as_str().to_string()), + None => None, + }, + None => { + // Alpha, rc Versions = like 3.10.0a3 + let python_regex = Regex::new(r"^(\d+\.\d+.\d+\w\d+)").unwrap(); + match python_regex.captures(&folder_name) { + Some(captures) => match captures.get(1) { + Some(version) => Some(version.as_str().to_string()), + None => None, + }, + None => None, + } + } + } + } + } +} + +fn get_pure_python_environment( + executable: &PathBuf, + path: &PathBuf, + manager: &Option, +) -> Option { + let version = get_pyenv_version(&path.file_name().unwrap().to_string_lossy().to_string())?; + Some(messaging::PythonEnvironment::new( + None, + None, + Some(executable.clone()), + messaging::PythonEnvironmentCategory::Pyenv, + Some(version), + Some(path.clone()), + manager.clone(), + Some(vec![executable + .clone() + .into_os_string() + .into_string() + .unwrap()]), + )) +} + +fn is_conda_environment(path: &PathBuf) -> bool { + if let Some(name) = path.file_name() { + let name = name.to_ascii_lowercase().to_string_lossy().to_string(); + return name.starts_with("anaconda") + || name.starts_with("miniconda") + || name.starts_with("miniforge"); + } + false +} + +fn get_virtual_env_environment( + executable: &PathBuf, + path: &PathBuf, + manager: &Option, +) -> Option { + let pyenv_cfg = find_and_parse_pyvenv_cfg(executable)?; + let folder_name = path.file_name().unwrap().to_string_lossy().to_string(); + Some(messaging::PythonEnvironment::new( + None, + Some(folder_name), + Some(executable.clone()), + messaging::PythonEnvironmentCategory::PyenvVirtualEnv, + Some(pyenv_cfg.version), + Some(path.clone()), + manager.clone(), + Some(vec![executable + .clone() + .into_os_string() + .into_string() + .unwrap()]), + )) +} + +pub fn list_pyenv_environments( + manager: &Option, + environment: &dyn known::Environment, + conda_locator: &mut dyn CondaLocator, +) -> Option> { + let pyenv_dir = get_pyenv_dir(environment)?; + let mut envs: Vec = vec![]; + let versions_dir = PathBuf::from(&pyenv_dir) + .join("versions") + .into_os_string() + .into_string() + .ok()?; + + for entry in fs::read_dir(&versions_dir).ok()?.filter_map(Result::ok) { + let path = entry.path(); + if !path.is_dir() { + continue; + } + if let Some(executable) = find_python_binary_path(&path) { + if let Some(env) = get_pure_python_environment(&executable, &path, manager) { + envs.push(env); + } else if let Some(env) = get_virtual_env_environment(&executable, &path, manager) { + envs.push(env); + } else if is_conda_environment(&path) { + if let Some(result) = conda_locator.find_in(&path) { + result.environments.iter().for_each(|e| { + envs.push(e.clone()); + }); + } + } + } + } + + Some(envs) +} + +pub struct PyEnv<'a> { + pub environment: &'a dyn Environment, + pub conda_locator: &'a mut dyn CondaLocator, +} + +impl PyEnv<'_> { + pub fn with<'a>( + environment: &'a impl Environment, + conda_locator: &'a mut impl CondaLocator, + ) -> PyEnv<'a> { + PyEnv { + environment, + conda_locator, + } + } +} + +impl Locator for PyEnv<'_> { + fn resolve(&self, _env: &PythonEnv) -> Option { + // We will find everything in gather + None + } + + fn find(&mut self) -> Option { + let pyenv_binary = get_pyenv_binary(self.environment)?; + let manager = messaging::EnvManager::new(pyenv_binary, None, EnvManagerType::Pyenv); + let mut environments: Vec = vec![]; + if let Some(envs) = + list_pyenv_environments(&Some(manager.clone()), self.environment, self.conda_locator) + { + for env in envs { + environments.push(env); + } + } + + Some(LocatorResult { + managers: vec![manager], + environments, + }) + } +} diff --git a/native_locator/src/utils.rs b/native_locator/src/utils.rs new file mode 100644 index 000000000000..c70efe9654ef --- /dev/null +++ b/native_locator/src/utils.rs @@ -0,0 +1,158 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +use crate::messaging::{EnvManager, PythonEnvironment}; +use regex::Regex; +use std::{ + fs, + path::{Path, PathBuf}, + process::Command, +}; + +#[derive(Debug)] +pub struct PythonEnv { + pub executable: PathBuf, + pub path: Option, + pub version: Option, +} + +impl PythonEnv { + pub fn new(executable: PathBuf, path: Option, version: Option) -> Self { + Self { + executable, + path, + version, + } + } +} + +#[derive(Debug)] +pub struct PyEnvCfg { + pub version: String, +} + +const PYVENV_CONFIG_FILE: &str = "pyvenv.cfg"; + +pub fn find_pyvenv_config_path(python_executable: &PathBuf) -> Option { + // Check if the pyvenv.cfg file is in the parent directory relative to the interpreter. + // env + // |__ pyvenv.cfg <--- check if this file exists + // |__ bin or Scripts + // |__ python <--- interpreterPath + let cfg = python_executable.parent()?.join(PYVENV_CONFIG_FILE); + if fs::metadata(&cfg).is_ok() { + return Some(cfg); + } + + // Check if the pyvenv.cfg file is in the directory as the interpreter. + // env + // |__ pyvenv.cfg <--- check if this file exists + // |__ python <--- interpreterPath + let cfg = python_executable + .parent()? + .parent()? + .join(PYVENV_CONFIG_FILE); + if fs::metadata(&cfg).is_ok() { + return Some(cfg); + } + + None +} + +pub fn find_and_parse_pyvenv_cfg(python_executable: &PathBuf) -> Option { + let cfg = find_pyvenv_config_path(&PathBuf::from(python_executable))?; + if !fs::metadata(&cfg).is_ok() { + return None; + } + + let contents = fs::read_to_string(&cfg).ok()?; + let version_regex = Regex::new(r"^version\s*=\s*(\d+\.\d+\.\d+)$").unwrap(); + let version_info_regex = Regex::new(r"^version_info\s*=\s*(\d+\.\d+\.\d+.*)$").unwrap(); + for line in contents.lines() { + if !line.contains("version") { + continue; + } + if let Some(captures) = version_regex.captures(line) { + if let Some(value) = captures.get(1) { + return Some(PyEnvCfg { + version: value.as_str().to_string(), + }); + } + } + if let Some(captures) = version_info_regex.captures(line) { + if let Some(value) = captures.get(1) { + return Some(PyEnvCfg { + version: value.as_str().to_string(), + }); + } + } + } + None +} + +pub fn get_version(python_executable: &PathBuf) -> Option { + if let Some(parent_folder) = python_executable.parent() { + if let Some(pyenv_cfg) = find_and_parse_pyvenv_cfg(&parent_folder.to_path_buf()) { + return Some(pyenv_cfg.version); + } + } + + let output = Command::new(python_executable) + .arg("-c") + .arg("import sys; print(sys.version)") + .output() + .ok()?; + let output = String::from_utf8(output.stdout).ok()?; + let output = output.trim(); + let output = output.split_whitespace().next().unwrap_or(output); + Some(output.to_string()) +} + +pub fn find_python_binary_path(env_path: &Path) -> Option { + let python_bin_name = if cfg!(windows) { + "python.exe" + } else { + "python" + }; + let path_1 = env_path.join("bin").join(python_bin_name); + let path_2 = env_path.join("Scripts").join(python_bin_name); + let path_3 = env_path.join(python_bin_name); + let paths = vec![path_1, path_2, path_3]; + paths.into_iter().find(|path| path.exists()) +} + +pub fn list_python_environments(path: &PathBuf) -> Option> { + let mut python_envs: Vec = vec![]; + for venv_dir in fs::read_dir(path).ok()? { + if let Ok(venv_dir) = venv_dir { + let venv_dir = venv_dir.path(); + if !venv_dir.is_dir() { + continue; + } + if let Some(executable) = find_python_binary_path(&venv_dir) { + python_envs.push(PythonEnv::new( + executable.clone(), + Some(venv_dir), + get_version(&executable), + )); + } + } + } + + Some(python_envs) +} + +pub fn get_environment_key(env: &PythonEnvironment) -> Option { + if let Some(ref path) = env.python_executable_path { + return Some(path.to_string_lossy().to_string()); + } + if let Some(ref path) = env.env_path { + return Some(path.to_string_lossy().to_string()); + } + + None +} + +pub fn get_environment_manager_key(env: &EnvManager) -> String { + return env.executable_path.to_string_lossy().to_string(); +} diff --git a/native_locator/src/venv.rs b/native_locator/src/venv.rs new file mode 100644 index 000000000000..94040a536989 --- /dev/null +++ b/native_locator/src/venv.rs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::{ + locator::{Locator, LocatorResult}, + messaging::PythonEnvironment, + utils::{self, PythonEnv}, +}; + +pub fn is_venv(env: &PythonEnv) -> bool { + // env path cannot be empty. + if env.path.is_none() { + return false; + } + return utils::find_pyvenv_config_path(&env.executable).is_some(); +} +pub struct Venv {} + +impl Venv { + pub fn new() -> Venv { + Venv {} + } +} + +impl Locator for Venv { + fn resolve(&self, env: &PythonEnv) -> Option { + if is_venv(&env) { + return Some(PythonEnvironment { + display_name: None, + name: Some( + env.path + .clone() + .expect("env.path can never be empty for venvs") + .file_name() + .unwrap() + .to_string_lossy() + .to_string(), + ), + python_executable_path: Some(env.executable.clone()), + version: env.version.clone(), + category: crate::messaging::PythonEnvironmentCategory::Venv, + env_path: env.path.clone(), + env_manager: None, + project_path: None, + python_run_command: Some(vec![env.executable.to_str().unwrap().to_string()]), + }); + } + None + } + + fn find(&mut self) -> Option { + // There are no common global locations for virtual environments. + // We expect the user of this class to call `is_compatible` + None + } +} diff --git a/native_locator/src/virtualenv.rs b/native_locator/src/virtualenv.rs new file mode 100644 index 000000000000..2a6909e63fa2 --- /dev/null +++ b/native_locator/src/virtualenv.rs @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::locator::{Locator, LocatorResult}; +use crate::messaging::PythonEnvironment; +use crate::utils::PythonEnv; + +pub fn is_virtualenv(env: &PythonEnv) -> bool { + if env.path.is_none() { + return false; + } + if let Some(file_path) = env.executable.parent() { + // Check if there are any activate.* files in the same directory as the interpreter. + // + // env + // |__ activate, activate.* <--- check if any of these files exist + // |__ python <--- interpreterPath + + // if let Some(parent_path) = PathBuf::from(env.) + // const directory = path.dirname(interpreterPath); + // const files = await fsapi.readdir(directory); + // const regex = /^activate(\.([A-z]|\d)+)?$/i; + if file_path.join("activate").exists() || file_path.join("activate.bat").exists() { + return true; + } + + // Support for activate.ps, etc. + match std::fs::read_dir(file_path) { + Ok(files) => { + for file in files { + if let Ok(file) = file { + if let Some(file_name) = file.file_name().to_str() { + if file_name.starts_with("activate") { + return true; + } + } + } + } + return false; + } + Err(_) => return false, + }; + } + + false +} + +pub struct VirtualEnv {} + +impl VirtualEnv { + pub fn new() -> VirtualEnv { + VirtualEnv {} + } +} + +impl Locator for VirtualEnv { + fn resolve(&self, env: &PythonEnv) -> Option { + if is_virtualenv(env) { + return Some(PythonEnvironment { + display_name: None, + name: Some( + env.path + .clone() + .expect("env.path can never be empty for virtualenvs") + .file_name() + .unwrap() + .to_string_lossy() + .to_string(), + ), + python_executable_path: Some(env.executable.clone()), + version: env.version.clone(), + category: crate::messaging::PythonEnvironmentCategory::VirtualEnv, + env_path: env.path.clone(), + env_manager: None, + project_path: None, + python_run_command: Some(vec![env.executable.to_str().unwrap().to_string()]), + }); + } + None + } + + fn find(&mut self) -> Option { + // There are no common global locations for virtual environments. + // We expect the user of this class to call `is_compatible` + None + } +} diff --git a/native_locator/src/virtualenvwrapper.rs b/native_locator/src/virtualenvwrapper.rs new file mode 100644 index 000000000000..d55a89e09dca --- /dev/null +++ b/native_locator/src/virtualenvwrapper.rs @@ -0,0 +1,124 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::locator::{Locator, LocatorResult}; +use crate::messaging::PythonEnvironment; +use crate::utils::list_python_environments; +use crate::virtualenv; +use crate::{known::Environment, utils::PythonEnv}; +use std::path::PathBuf; + +#[cfg(windows)] +fn get_default_virtualenvwrapper_path(environment: &dyn Environment) -> Option { + // In Windows, the default path for WORKON_HOME is %USERPROFILE%\Envs. + // If 'Envs' is not available we should default to '.virtualenvs'. Since that + // is also valid for windows. + if let Some(home) = environment.get_user_home() { + let home = PathBuf::from(home).join("Envs"); + if home.exists() { + return Some(home); + } + let home = PathBuf::from(home).join("virtualenvs"); + if home.exists() { + return Some(home); + } + } + None +} + +#[cfg(unix)] +fn get_default_virtualenvwrapper_path(environment: &dyn Environment) -> Option { + if let Some(home) = environment.get_user_home() { + let home = PathBuf::from(home).join("virtualenvs"); + if home.exists() { + return Some(home); + } + } + None +} + +fn get_work_on_home_path(environment: &dyn Environment) -> Option { + // The WORKON_HOME variable contains the path to the root directory of all virtualenvwrapper environments. + // If the interpreter path belongs to one of them then it is a virtualenvwrapper type of environment. + if let Some(work_on_home) = environment.get_env_var("WORKON_HOME".to_string()) { + if let Ok(work_on_home) = std::fs::canonicalize(work_on_home) { + if work_on_home.exists() { + return Some(work_on_home); + } + } + } + get_default_virtualenvwrapper_path(environment) +} + +pub fn is_virtualenvwrapper(env: &PythonEnv, environment: &dyn Environment) -> bool { + if env.path.is_none() { + return false; + } + // For environment to be a virtualenvwrapper based it has to follow these two rules: + // 1. It should be in a sub-directory under the WORKON_HOME + // 2. It should be a valid virtualenv environment + if let Some(work_on_home_dir) = get_work_on_home_path(environment) { + if env.executable.starts_with(&work_on_home_dir) && virtualenv::is_virtualenv(env) { + return true; + } + } + + false +} + +pub struct VirtualEnvWrapper<'a> { + pub environment: &'a dyn Environment, +} + +impl VirtualEnvWrapper<'_> { + pub fn with<'a>(environment: &'a impl Environment) -> VirtualEnvWrapper { + VirtualEnvWrapper { environment } + } +} + +impl Locator for VirtualEnvWrapper<'_> { + fn resolve(&self, env: &PythonEnv) -> Option { + if is_virtualenvwrapper(env, self.environment) { + return Some(PythonEnvironment { + display_name: None, + name: Some( + env.path + .clone() + .expect("env.path cannot be empty for virtualenv rapper") + .file_name() + .unwrap() + .to_string_lossy() + .to_string(), + ), + python_executable_path: Some(env.executable.clone()), + version: env.version.clone(), + category: crate::messaging::PythonEnvironmentCategory::Venv, + env_path: env.path.clone(), + env_manager: None, + project_path: None, + python_run_command: Some(vec![env.executable.to_str().unwrap().to_string()]), + }); + } + None + } + + fn find(&mut self) -> Option { + let work_on_home = get_work_on_home_path(self.environment)?; + let envs = list_python_environments(&work_on_home)?; + let mut environments: Vec = vec![]; + envs.iter().for_each(|env| { + if let Some(env) = self.resolve(env) { + environments.push(env); + } + }); + + if environments.is_empty() { + None + } else { + Some(LocatorResult { + managers: vec![], + environments, + }) + } + } +} diff --git a/native_locator/src/windows_registry.rs b/native_locator/src/windows_registry.rs new file mode 100644 index 000000000000..4f8e97710fc2 --- /dev/null +++ b/native_locator/src/windows_registry.rs @@ -0,0 +1,156 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#[cfg(windows)] +use crate::conda::CondaLocator; +#[cfg(windows)] +use crate::locator::{Locator, LocatorResult}; +#[cfg(windows)] +use crate::messaging::EnvManager; +#[cfg(windows)] +use crate::messaging::{PythonEnvironment, PythonEnvironmentCategory}; +#[cfg(windows)] +use crate::utils::PythonEnv; +#[cfg(windows)] +use std::path::PathBuf; +#[cfg(windows)] +use winreg::RegKey; + +#[cfg(windows)] +fn get_registry_pythons_from_key(hk: &RegKey, company: &str) -> Option> { + let python_key = hk.open_subkey("Software\\Python").ok()?; + let company_key = python_key.open_subkey(company).ok()?; + + let mut pythons = vec![]; + for key in company_key.enum_keys().filter_map(Result::ok) { + if let Some(key) = company_key.open_subkey(key).ok() { + if let Some(install_path_key) = key.open_subkey("InstallPath").ok() { + let env_path: String = install_path_key.get_value("").ok().unwrap_or_default(); + let env_path = PathBuf::from(env_path); + let env_path = if env_path.exists() { + Some(env_path) + } else { + None + }; + let executable: String = install_path_key + .get_value("ExecutablePath") + .ok() + .unwrap_or_default(); + if executable.len() == 0 { + continue; + } + let executable = PathBuf::from(executable); + if !executable.exists() { + continue; + } + let version: String = key.get_value("Version").ok().unwrap_or_default(); + let display_name: String = key.get_value("DisplayName").ok().unwrap_or_default(); + + let env = PythonEnvironment::new( + Some(display_name), + None, + Some(executable.clone()), + PythonEnvironmentCategory::WindowsRegistry, + if version.len() > 0 { + Some(version) + } else { + None + }, + env_path, + None, + Some(vec![executable.to_string_lossy().to_string()]), + ); + pythons.push(env); + } + } + } + + Some(pythons) +} + +#[cfg(windows)] +pub fn get_registry_pythons(company: &str) -> Option> { + let hklm = winreg::RegKey::predef(winreg::enums::HKEY_LOCAL_MACHINE); + let hkcu = winreg::RegKey::predef(winreg::enums::HKEY_CURRENT_USER); + + let mut pythons = vec![]; + if let Some(hklm_pythons) = get_registry_pythons_from_key(&hklm, company) { + pythons.extend(hklm_pythons); + } + if let Some(hkcu_pythons) = get_registry_pythons_from_key(&hkcu, company) { + pythons.extend(hkcu_pythons); + } + Some(pythons) +} + +#[cfg(windows)] +pub fn get_registry_pythons_anaconda(conda_locator: &mut dyn CondaLocator) -> LocatorResult { + let hklm = winreg::RegKey::predef(winreg::enums::HKEY_LOCAL_MACHINE); + let hkcu = winreg::RegKey::predef(winreg::enums::HKEY_CURRENT_USER); + + let mut pythons = vec![]; + if let Some(hklm_pythons) = get_registry_pythons_from_key(&hklm, "ContinuumAnalytics") { + pythons.extend(hklm_pythons); + } + if let Some(hkcu_pythons) = get_registry_pythons_from_key(&hkcu, "ContinuumAnalytics") { + pythons.extend(hkcu_pythons); + } + + let mut environments: Vec = vec![]; + let mut managers: Vec = vec![]; + + for env in pythons.iter() { + if let Some(env_path) = env.clone().env_path { + if let Some(mut result) = conda_locator.find_in(&env_path) { + environments.append(&mut result.environments); + managers.append(&mut result.managers); + } + } + } + + LocatorResult { + managers, + environments, + } +} + +#[cfg(windows)] +pub struct WindowsRegistry<'a> { + pub conda_locator: &'a mut dyn CondaLocator, +} + +#[cfg(windows)] +impl WindowsRegistry<'_> { + #[allow(dead_code)] + pub fn with<'a>(conda_locator: &'a mut impl CondaLocator) -> WindowsRegistry<'a> { + WindowsRegistry { conda_locator } + } +} + +#[cfg(windows)] +impl Locator for WindowsRegistry<'_> { + fn resolve(&self, _env: &PythonEnv) -> Option { + None + } + + fn find(&mut self) -> Option { + let mut environments: Vec = vec![]; + let mut managers: Vec = vec![]; + + let mut result = get_registry_pythons("PythonCore").unwrap_or_default(); + environments.append(&mut result); + + let mut result = get_registry_pythons_anaconda(self.conda_locator) ; + environments.append(&mut result.environments); + managers.append(&mut result.managers); + + if environments.is_empty() && managers.is_empty() { + None + } else { + Some(LocatorResult { + managers, + environments, + }) + } + } +} diff --git a/native_locator/src/windows_store.rs b/native_locator/src/windows_store.rs new file mode 100644 index 000000000000..f08622d08127 --- /dev/null +++ b/native_locator/src/windows_store.rs @@ -0,0 +1,151 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#[cfg(windows)] +use crate::known; +#[cfg(windows)] +use crate::known::Environment; +#[cfg(windows)] +use crate::locator::{Locator, LocatorResult}; +#[cfg(windows)] +use crate::messaging::PythonEnvironment; +#[cfg(windows)] +use crate::utils::PythonEnv; +#[cfg(windows)] +use std::path::Path; +#[cfg(windows)] +use std::path::PathBuf; +#[cfg(windows)] +use winreg::RegKey; + +#[cfg(windows)] +pub fn is_windows_python_executable(path: &PathBuf) -> bool { + let name = path.file_name().unwrap().to_string_lossy().to_lowercase(); + // TODO: Is it safe to assume the number 3? + name.starts_with("python3.") && name.ends_with(".exe") +} + +#[cfg(windows)] +fn list_windows_store_python_executables( + environment: &dyn known::Environment, +) -> Option> { + let mut python_envs: Vec = vec![]; + let home = environment.get_user_home()?; + let apps_path = Path::new(&home) + .join("AppData") + .join("Local") + .join("Microsoft") + .join("WindowsApps"); + let hkcu = winreg::RegKey::predef(winreg::enums::HKEY_CURRENT_USER); + for file in std::fs::read_dir(apps_path).ok()?.filter_map(Result::ok) { + let path = file.path(); + if let Some(name) = path.file_name() { + let exe = path.join("python.exe"); + if name + .to_str() + .unwrap_or_default() + .starts_with("PythonSoftwareFoundation.Python.") + && exe.is_file() + && exe.exists() + { + if let Some(result) = + get_package_display_name_and_location(name.to_string_lossy().to_string(), &hkcu) + { + let env = PythonEnvironment { + display_name: Some(result.display_name), + name: None, + python_executable_path: Some(exe.clone()), + version: None, + category: crate::messaging::PythonEnvironmentCategory::WindowsStore, + env_path: Some(PathBuf::from(result.env_path.clone())), + env_manager: None, + project_path: None, + python_run_command: Some(vec![exe.to_string_lossy().to_string()]), + }; + python_envs.push(env); + } + } + } + } + + Some(python_envs) +} + +#[cfg(windows)] +fn get_package_full_name_from_registry(name: String, hkcu: &RegKey) -> Option { + let key = format!("Software\\Classes\\Local Settings\\Software\\Microsoft\\Windows\\CurrentVersion\\AppModel\\SystemAppData\\{}\\Schemas", name); + let package_key = hkcu.open_subkey(key).ok()?; + let value = package_key.get_value("PackageFullName").unwrap_or_default(); + Some(value) +} + +#[derive(Debug)] +#[cfg(windows)] +struct StorePythonInfo { + display_name: String, + env_path: String, +} + +#[cfg(windows)] +fn get_package_display_name_and_location(name: String, hkcu: &RegKey) -> Option { + if let Some(name) = get_package_full_name_from_registry(name, &hkcu) { + let key = format!("Software\\Classes\\Local Settings\\Software\\Microsoft\\Windows\\CurrentVersion\\AppModel\\Repository\\Packages\\{}", name); + let package_key = hkcu.open_subkey(key).ok()?; + let display_name = package_key.get_value("DisplayName").ok()?; + let env_path = package_key.get_value("PackageRootFolder").ok()?; + + let regex = regex::Regex::new("PythonSoftwareFoundation.Python.((\\d+\\.?)*)_.*").ok()?; + + return Some(StorePythonInfo { + display_name, + env_path, + }); + } + None +} + +#[cfg(windows)] +pub struct WindowsStore<'a> { + pub environment: &'a dyn Environment, +} + +#[cfg(windows)] +impl WindowsStore<'_> { + #[allow(dead_code)] + pub fn with<'a>(environment: &'a impl Environment) -> WindowsStore { + WindowsStore { environment } + } +} + +#[cfg(windows)] +impl Locator for WindowsStore<'_> { + fn resolve(&self, env: &PythonEnv) -> Option { + if is_windows_python_executable(&env.executable) { + return Some(PythonEnvironment { + display_name: None, + name: None, + python_executable_path: Some(env.executable.clone()), + version: None, + category: crate::messaging::PythonEnvironmentCategory::WindowsStore, + env_path: None, + env_manager: None, + project_path: None, + python_run_command: Some(vec![env.executable.to_str().unwrap().to_string()]), + }); + } + None + } + + fn find(&mut self) -> Option { + let environments = list_windows_store_python_executables(self.environment)?; + + if environments.is_empty() { + None + } else { + Some(LocatorResult { + managers: vec![], + environments, + }) + } + } +} diff --git a/native_locator/tests/common.rs b/native_locator/tests/common.rs new file mode 100644 index 000000000000..bf4c54617f16 --- /dev/null +++ b/native_locator/tests/common.rs @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use python_finder::known::Environment; +use serde_json::Value; +use std::{collections::HashMap, path::PathBuf}; + +#[allow(dead_code)] +pub fn test_file_path(paths: &[&str]) -> PathBuf { + let mut root = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + + paths.iter().for_each(|p| root.push(p)); + + root +} + +#[allow(dead_code)] +pub fn join_test_paths(paths: &[&str]) -> PathBuf { + let path: PathBuf = paths.iter().map(|p| p.to_string()).collect(); + path +} + +pub trait TestMessages { + fn get_messages(&self) -> Vec; +} + +pub struct TestEnvironment { + vars: HashMap, + home: Option, + root: Option, + globals_locations: Vec, +} +#[allow(dead_code)] +pub fn create_test_environment( + vars: HashMap, + home: Option, + globals_locations: Vec, + root: Option, +) -> TestEnvironment { + impl Environment for TestEnvironment { + fn get_env_var(&self, key: String) -> Option { + self.vars.get(&key).cloned() + } + fn get_root(&self) -> Option { + self.root.clone() + } + fn get_user_home(&self) -> Option { + self.home.clone() + } + fn get_know_global_search_locations(&self) -> Vec { + self.globals_locations.clone() + } + } + TestEnvironment { + vars, + home, + root, + globals_locations, + } +} + +fn compare_json(expected: &Value, actual: &Value) -> bool { + if expected == actual { + return true; + } + + if expected.is_object() { + if expected.as_object().is_none() && actual.as_object().is_none() { + return true; + } + + if expected.as_object().is_none() && actual.as_object().is_some() { + return false; + } + if expected.as_object().is_some() && actual.as_object().is_none() { + return false; + } + + let expected = expected.as_object().unwrap(); + let actual = actual.as_object().unwrap(); + + for (key, value) in expected.iter() { + if !actual.contains_key(key) { + return false; + } + if !compare_json(value, actual.get(key).unwrap()) { + return false; + } + } + return true; + } + + if expected.is_array() { + let expected = expected.as_array().unwrap(); + let actual = actual.as_array().unwrap(); + + if expected.len() != actual.len() { + return false; + } + + for (i, value) in expected.iter().enumerate() { + if !compare_json(value, actual.get(i).unwrap()) { + return false; + } + } + return true; + } + + false +} + +#[allow(dead_code)] +pub fn assert_messages(expected_json: &[Value], actual_json: &[Value]) { + let mut expected_json = expected_json.to_vec(); + assert_eq!( + expected_json.len(), + actual_json.len(), + "Incorrect number of messages" + ); + + if expected_json.len() == 0 { + return; + } + + // Ignore the order of the json items when comparing. + for actual in actual_json.iter() { + let mut valid_index: Option = None; + for (i, expected) in expected_json.iter().enumerate() { + if !compare_json(expected, &actual) { + continue; + } + + // Ensure we verify using standard assert_eq!, just in case the code is faulty.. + valid_index = Some(i); + assert_eq!(expected, actual); + } + if let Some(index) = valid_index { + // This is to ensure we don't compare the same item twice. + expected_json.remove(index); + } else { + // Use traditional assert so we can see the fully output in the test results. + assert_eq!(&expected_json[0], actual); + } + } +} diff --git a/native_locator/tests/common_python_test.rs b/native_locator/tests/common_python_test.rs new file mode 100644 index 000000000000..d7d71fd5cce1 --- /dev/null +++ b/native_locator/tests/common_python_test.rs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +mod common; + +#[test] +#[cfg(unix)] +fn find_python_in_path_this() { + use crate::common::{ + assert_messages, create_test_environment, join_test_paths, test_file_path, + }; + use python_finder::{common_python, locator::Locator, messaging::PythonEnvironment}; + use serde_json::json; + use std::collections::HashMap; + + let user_home = test_file_path(&["tests/unix/known/user_home"]); + let unix_python_exe = join_test_paths(&[user_home.clone().to_str().unwrap(), "python"]); + + let known = create_test_environment( + HashMap::from([( + "PATH".to_string(), + user_home.clone().to_string_lossy().to_string(), + )]), + Some(user_home.clone()), + Vec::new(), + None, + ); + + let mut locator = common_python::PythonOnPath::with(&known); + let result = locator.find().unwrap(); + + assert_eq!(result.environments.len(), 1); + + let env = PythonEnvironment { + display_name: None, + env_manager: None, + project_path: None, + name: None, + python_executable_path: Some(unix_python_exe.clone()), + category: python_finder::messaging::PythonEnvironmentCategory::System, + version: None, + python_run_command: Some(vec![unix_python_exe.clone().to_str().unwrap().to_string()]), + env_path: Some(user_home.clone()), + }; + assert_messages( + &[json!(env)], + &result + .environments + .iter() + .map(|e| json!(e)) + .collect::>(), + ); +} diff --git a/native_locator/tests/conda_test.rs b/native_locator/tests/conda_test.rs new file mode 100644 index 000000000000..397ed42b1de6 --- /dev/null +++ b/native_locator/tests/conda_test.rs @@ -0,0 +1,290 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +mod common; + +#[test] +#[cfg(unix)] +fn does_not_find_any_conda_envs() { + use crate::common::create_test_environment; + use python_finder::{conda, locator::Locator}; + use std::{collections::HashMap, path::PathBuf}; + + let known = create_test_environment( + HashMap::from([("PATH".to_string(), "".to_string())]), + Some(PathBuf::from("SOME_BOGUS_HOME_DIR")), + Vec::new(), + None, + ); + + let mut locator = conda::Conda::with(&known); + let result = locator.find(); + + assert_eq!(result.is_none(), true); +} + +#[test] +#[cfg(unix)] +fn no_paths_from_conda_rc_if_conda_rc_does_not_exist() { + use crate::common::{create_test_environment, test_file_path}; + use python_finder::conda::get_conda_environment_paths_from_conda_rc; + use std::collections::HashMap; + + let user_home = test_file_path(&["tests/unix/no_conda_rc/user_home"]); + let root = test_file_path(&["tests/unix/no_conda_rc/root"]); + + let known = create_test_environment( + HashMap::from([("PATH".to_string(), "".to_string())]), + Some(user_home), + Vec::new(), + Some(root), + ); + + let result = get_conda_environment_paths_from_conda_rc(&known); + + assert_eq!(result.len(), 0); +} + +#[test] +#[cfg(unix)] +fn paths_from_conda_rc() { + use crate::common::{create_test_environment, test_file_path}; + use python_finder::conda::get_conda_environment_paths_from_conda_rc; + use std::{collections::HashMap, fs, path::PathBuf}; + + fn create_conda_rc(file: &PathBuf, paths: &Vec) { + use std::fs::File; + use std::io::Write; + let mut file = File::create(file).unwrap(); + + writeln!(file, "envs_dirs:").unwrap(); + for path in paths { + writeln!(file, " - {}", path.to_string_lossy()).unwrap(); + } + } + + fn test_with(conda_rc_file: &PathBuf) { + let home = test_file_path(&["tests/unix/conda_rc/user_home"]); + let root = test_file_path(&["tests/unix/conda_rc/root"]); + let conda_dir = home.join(".conda"); + let conda_envs = conda_dir.join("envs"); + + let known = create_test_environment( + HashMap::from([("PATH".to_string(), "".to_string())]), + Some(home.clone()), + Vec::new(), + Some(root.clone()), + ); + fs::remove_dir_all(home.clone()).unwrap_or_default(); + fs::remove_dir_all(root.clone()).unwrap_or_default(); + + fs::create_dir_all(home.clone()).unwrap_or_default(); + fs::create_dir_all(root.clone()).unwrap_or_default(); + fs::create_dir_all(conda_envs.clone()).unwrap_or_default(); + fs::create_dir_all(conda_rc_file.parent().unwrap()).unwrap_or_default(); + + create_conda_rc(conda_rc_file, &vec![conda_dir.clone()]); + + let result = get_conda_environment_paths_from_conda_rc(&known); + assert_eq!(result.len(), 1); + assert_eq!(result[0], conda_envs); + + fs::remove_dir_all(home.clone()).unwrap_or_default(); + fs::remove_dir_all(root.clone()).unwrap_or_default(); + } + + let home = test_file_path(&["tests/unix/conda_rc/user_home"]); + let root = test_file_path(&["tests/unix/conda_rc/root"]); + + test_with(&root.join("etc/conda/.condarc")); + test_with(&home.join(".condarc")); +} + +#[test] +#[cfg(unix)] +fn find_conda_exe_and_empty_envs() { + use crate::common::{create_test_environment, join_test_paths, test_file_path}; + use python_finder::messaging::{EnvManager, EnvManagerType}; + use python_finder::{conda, locator::Locator}; + use serde_json::json; + use std::collections::HashMap; + let user_home = test_file_path(&["tests/unix/conda_without_envs/user_home"]); + let conda_dir = test_file_path(&["tests/unix/conda_without_envs/user_home"]); + + let known = create_test_environment( + HashMap::from([( + "PATH".to_string(), + conda_dir.clone().to_str().unwrap().to_string(), + )]), + Some(user_home), + Vec::new(), + None, + ); + + let mut locator = conda::Conda::with(&known); + let result = locator.find().unwrap(); + let managers = result.managers; + assert_eq!(managers.len(), 1); + + let conda_exe = join_test_paths(&[ + conda_dir.clone().to_str().unwrap(), + "anaconda3", + "bin", + "conda", + ]); + let expected_conda_manager = EnvManager { + executable_path: conda_exe.clone(), + version: Some("4.0.2".to_string()), + tool: EnvManagerType::Conda, + }; + assert_eq!(managers.len(), 1); + assert_eq!(json!(expected_conda_manager), json!(managers[0])); +} + +#[test] +#[cfg(unix)] +fn find_conda_from_custom_install_location() { + use crate::common::{create_test_environment, test_file_path}; + use python_finder::messaging::{EnvManager, EnvManagerType, PythonEnvironment}; + use python_finder::{conda, locator::Locator}; + use serde_json::json; + use std::collections::HashMap; + use std::fs; + + let home = test_file_path(&["tests/unix/conda_custom_install_path/user_home"]); + let conda_dir = + test_file_path(&["tests/unix/conda_custom_install_path/user_home/some_location/anaconda3"]); + let environments_txt = + test_file_path(&["tests/unix/conda_custom_install_path/user_home/.conda/environments.txt"]); + + fs::create_dir_all(environments_txt.parent().unwrap()).unwrap_or_default(); + fs::write( + environments_txt.clone(), + format!("{}", conda_dir.clone().to_str().unwrap().to_string()), + ) + .unwrap(); + + let known = create_test_environment(HashMap::new(), Some(home), Vec::new(), None); + + let mut locator = conda::Conda::with(&known); + let result = locator.find().unwrap(); + + assert_eq!(result.managers.len(), 1); + assert_eq!(result.environments.len(), 1); + + let conda_exe = conda_dir.clone().join("bin").join("conda"); + let expected_conda_manager = EnvManager { + executable_path: conda_exe.clone(), + version: Some("4.0.2".to_string()), + tool: EnvManagerType::Conda, + }; + assert_eq!(json!(expected_conda_manager), json!(result.managers[0])); + + let expected_conda_env = PythonEnvironment { + display_name: None, + name: None, + project_path: None, + python_executable_path: Some(conda_dir.clone().join("bin").join("python")), + category: python_finder::messaging::PythonEnvironmentCategory::Conda, + version: Some("10.0.1".to_string()), + env_path: Some(conda_dir.clone()), + env_manager: Some(expected_conda_manager.clone()), + python_run_command: Some(vec![ + conda_exe.clone().to_str().unwrap().to_string(), + "run".to_string(), + "-p".to_string(), + conda_dir.to_string_lossy().to_string(), + "python".to_string(), + ]), + }; + assert_eq!(json!(expected_conda_env), json!(result.environments[0])); + + // Reset environments.txt + fs::write(environments_txt.clone(), "").unwrap(); +} + +#[test] +#[cfg(unix)] +fn finds_two_conda_envs_from_known_location() { + use crate::common::{ + assert_messages, create_test_environment, join_test_paths, test_file_path, + }; + use python_finder::messaging::{EnvManager, EnvManagerType, PythonEnvironment}; + use python_finder::{conda, locator::Locator}; + use serde_json::json; + use std::collections::HashMap; + + let home = test_file_path(&["tests/unix/conda/user_home"]); + let conda_dir = test_file_path(&["tests/unix/conda/user_home/anaconda3"]); + let conda_1 = join_test_paths(&[conda_dir.clone().to_str().unwrap(), "envs/one"]); + let conda_2 = join_test_paths(&[conda_dir.clone().to_str().unwrap(), "envs/two"]); + + let known = create_test_environment( + HashMap::from([( + "PATH".to_string(), + conda_dir.clone().to_str().unwrap().to_string(), + )]), + Some(home), + Vec::new(), + None, + ); + + let mut locator = conda::Conda::with(&known); + let result = locator.find().unwrap(); + + let managers = result.managers; + let environments = result.environments; + assert_eq!(managers.len(), 1); + + let conda_exe = join_test_paths(&[conda_dir.clone().to_str().unwrap(), "bin", "conda"]); + let conda_1_exe = join_test_paths(&[conda_1.clone().to_str().unwrap(), "python"]); + let conda_2_exe = join_test_paths(&[conda_2.clone().to_str().unwrap(), "python"]); + + let expected_conda_manager = EnvManager { + executable_path: conda_exe.clone(), + version: Some("4.0.2".to_string()), + tool: EnvManagerType::Conda, + }; + + assert_eq!(managers.len(), 1); + assert_eq!(json!(expected_conda_manager), json!(managers[0])); + + let expected_conda_1 = PythonEnvironment { + display_name: None, + name: Some("one".to_string()), + project_path: None, + python_executable_path: Some(conda_1_exe.clone()), + category: python_finder::messaging::PythonEnvironmentCategory::Conda, + version: Some("10.0.1".to_string()), + env_path: Some(conda_1.clone()), + env_manager: Some(expected_conda_manager.clone()), + python_run_command: Some(vec![ + conda_exe.clone().to_str().unwrap().to_string(), + "run".to_string(), + "-n".to_string(), + "one".to_string(), + "python".to_string(), + ]), + }; + let expected_conda_2 = PythonEnvironment { + display_name: None, + name: Some("two".to_string()), + project_path: None, + python_executable_path: Some(conda_2_exe.clone()), + category: python_finder::messaging::PythonEnvironmentCategory::Conda, + version: None, + env_path: Some(conda_2.clone()), + env_manager: Some(expected_conda_manager.clone()), + python_run_command: Some(vec![ + conda_exe.clone().to_str().unwrap().to_string(), + "run".to_string(), + "-n".to_string(), + "two".to_string(), + "python".to_string(), + ]), + }; + assert_messages( + &[json!(expected_conda_1), json!(expected_conda_2)], + &environments.iter().map(|e| json!(e)).collect::>(), + ); +} diff --git a/native_locator/tests/pyenv_test.rs b/native_locator/tests/pyenv_test.rs new file mode 100644 index 000000000000..87761114089d --- /dev/null +++ b/native_locator/tests/pyenv_test.rs @@ -0,0 +1,240 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +mod common; + +#[test] +#[cfg(unix)] +fn does_not_find_any_pyenv_envs() { + use crate::common::create_test_environment; + use python_finder::{conda::Conda, locator::Locator, pyenv}; + use std::{collections::HashMap, path::PathBuf}; + + let known = create_test_environment( + HashMap::new(), + Some(PathBuf::from("SOME_BOGUS_HOME_DIR")), + Vec::new(), + None, + ); + + let mut conda = Conda::with(&known); + let mut locator = pyenv::PyEnv::with(&known, &mut conda); + let result = locator.find(); + + assert_eq!(result.is_none(), true); +} + +#[test] +#[cfg(unix)] +fn does_not_find_any_pyenv_envs_even_with_pyenv_installed() { + use crate::common::{ + assert_messages, create_test_environment, join_test_paths, test_file_path, + }; + use python_finder::pyenv; + use python_finder::{conda::Conda, locator::Locator}; + use serde_json::json; + use std::{collections::HashMap, path::PathBuf}; + + let home = test_file_path(&["tests", "unix", "pyenv_without_envs", "user_home"]); + let homebrew_bin = test_file_path(&[ + "tests", + "unix", + "pyenv_without_envs", + "home", + "opt", + "homebrew", + "bin", + ]); + let pyenv_exe = join_test_paths(&[homebrew_bin.to_str().unwrap(), "pyenv"]); + let known = create_test_environment( + HashMap::new(), + Some(home.clone()), + vec![PathBuf::from(homebrew_bin)], + None, + ); + + let mut conda = Conda::with(&known); + let mut locator = pyenv::PyEnv::with(&known, &mut conda); + let result = locator.find().unwrap(); + + let managers = result.managers; + assert_eq!(managers.len(), 1); + + let expected_json = json!({"executablePath":pyenv_exe,"version":null, "tool": "pyenv"}); + assert_messages( + &[expected_json], + &managers.iter().map(|e| json!(e)).collect::>(), + ) +} + +#[test] +#[cfg(unix)] +fn find_pyenv_envs() { + use crate::common::{ + assert_messages, create_test_environment, join_test_paths, + test_file_path, + }; + use python_finder::conda::Conda; + use python_finder::locator::Locator; + use python_finder::{ + messaging::{EnvManager, EnvManagerType, PythonEnvironment}, + pyenv, + }; + use serde_json::json; + use std::{collections::HashMap, path::PathBuf}; + + let home = test_file_path(&["tests", "unix", "pyenv", "user_home"]); + let homebrew_bin = + test_file_path(&["tests", "unix", "pyenv", "home", "opt", "homebrew", "bin"]); + let pyenv_exe = join_test_paths(&[homebrew_bin.to_str().unwrap(), "pyenv"]); + let known = create_test_environment( + HashMap::new(), + Some(home.clone()), + vec![PathBuf::from(homebrew_bin)], + None, + ); + + let mut conda = Conda::with(&known); + let mut locator = pyenv::PyEnv::with(&known, &mut conda); + let result = locator.find().unwrap(); + + assert_eq!(result.managers.len(), 1); + + let expected_manager = EnvManager { + executable_path: pyenv_exe.clone(), + version: None, + tool: EnvManagerType::Pyenv, + }; + assert_eq!(json!(expected_manager), json!(result.managers[0])); + + let expected_3_9_9 = json!(PythonEnvironment { + display_name: None, + project_path: None, + name: None, + python_executable_path: Some(join_test_paths(&[ + home.to_str().unwrap(), + ".pyenv/versions/3.9.9/bin/python" + ])), + python_run_command: Some(vec![join_test_paths(&[ + home.to_str().unwrap(), + ".pyenv/versions/3.9.9/bin/python" + ]) + .to_str() + .unwrap() + .to_string()]), + category: python_finder::messaging::PythonEnvironmentCategory::Pyenv, + version: Some("3.9.9".to_string()), + env_path: Some(join_test_paths(&[ + home.to_str().unwrap(), + ".pyenv/versions/3.9.9" + ])), + env_manager: Some(expected_manager.clone()) + }); + let expected_virtual_env = PythonEnvironment { + display_name: None, + project_path: None, + name: Some("my-virtual-env".to_string()), + python_executable_path: Some(join_test_paths(&[ + home.to_str().unwrap(), + ".pyenv/versions/my-virtual-env/bin/python", + ])), + python_run_command: Some(vec![join_test_paths(&[ + home.to_str().unwrap(), + ".pyenv/versions/my-virtual-env/bin/python", + ]) + .to_str() + .unwrap() + .to_string()]), + category: python_finder::messaging::PythonEnvironmentCategory::PyenvVirtualEnv, + version: Some("3.10.13".to_string()), + env_path: Some(join_test_paths(&[ + home.to_str().unwrap(), + ".pyenv/versions/my-virtual-env", + ])), + env_manager: Some(expected_manager.clone()), + }; + let expected_3_12_1 = PythonEnvironment { + display_name: None, + project_path: None, + name: None, + python_executable_path: Some(join_test_paths(&[ + home.to_str().unwrap(), + ".pyenv/versions/3.12.1/bin/python", + ])), + python_run_command: Some(vec![join_test_paths(&[ + home.to_str().unwrap(), + ".pyenv/versions/3.12.1/bin/python", + ]) + .to_str() + .unwrap() + .to_string()]), + category: python_finder::messaging::PythonEnvironmentCategory::Pyenv, + version: Some("3.12.1".to_string()), + env_path: Some(join_test_paths(&[ + home.to_str().unwrap(), + ".pyenv/versions/3.12.1", + ])), + env_manager: Some(expected_manager.clone()), + }; + let expected_3_13_dev = PythonEnvironment { + display_name: None, + project_path: None, + name: None, + python_executable_path: Some(join_test_paths(&[ + home.to_str().unwrap(), + ".pyenv/versions/3.13-dev/bin/python", + ])), + python_run_command: Some(vec![join_test_paths(&[ + home.to_str().unwrap(), + ".pyenv/versions/3.13-dev/bin/python", + ]) + .to_str() + .unwrap() + .to_string()]), + category: python_finder::messaging::PythonEnvironmentCategory::Pyenv, + version: Some("3.13-dev".to_string()), + env_path: Some(join_test_paths(&[ + home.to_str().unwrap(), + ".pyenv/versions/3.13-dev", + ])), + env_manager: Some(expected_manager.clone()), + }; + let expected_3_12_1a3 = PythonEnvironment { + display_name: None, + project_path: None, + name: None, + python_executable_path: Some(join_test_paths(&[ + home.to_str().unwrap(), + ".pyenv/versions/3.12.1a3/bin/python", + ])), + python_run_command: Some(vec![join_test_paths(&[ + home.to_str().unwrap(), + ".pyenv/versions/3.12.1a3/bin/python", + ]) + .to_str() + .unwrap() + .to_string()]), + category: python_finder::messaging::PythonEnvironmentCategory::Pyenv, + version: Some("3.12.1a3".to_string()), + env_path: Some(join_test_paths(&[ + home.to_str().unwrap(), + ".pyenv/versions/3.12.1a3", + ])), + env_manager: Some(expected_manager.clone()), + }; + + assert_messages( + &[ + json!(expected_3_9_9), + json!(expected_virtual_env), + json!(expected_3_12_1), + json!(expected_3_13_dev), + json!(expected_3_12_1a3), + ], + &result + .environments + .iter() + .map(|e| json!(e)) + .collect::>(), + ) +} diff --git a/native_locator/tests/unix/conda/user_home/.conda/environments.txt b/native_locator/tests/unix/conda/user_home/.conda/environments.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/conda/user_home/anaconda3/bin/conda b/native_locator/tests/unix/conda/user_home/anaconda3/bin/conda new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/conda/user_home/anaconda3/conda-meta/conda-4.0.2-pyhd3eb1b0_0.json b/native_locator/tests/unix/conda/user_home/anaconda3/conda-meta/conda-4.0.2-pyhd3eb1b0_0.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/conda/user_home/anaconda3/conda-meta/python-10.0.1-hdf0ec26_0_cpython.json b/native_locator/tests/unix/conda/user_home/anaconda3/conda-meta/python-10.0.1-hdf0ec26_0_cpython.json new file mode 100644 index 000000000000..23127993ac05 --- /dev/null +++ b/native_locator/tests/unix/conda/user_home/anaconda3/conda-meta/python-10.0.1-hdf0ec26_0_cpython.json @@ -0,0 +1 @@ +10.1.1 diff --git a/native_locator/tests/unix/conda/user_home/anaconda3/conda-meta/python-slugify-5.0.2-pyhd3eb1b0_0.json b/native_locator/tests/unix/conda/user_home/anaconda3/conda-meta/python-slugify-5.0.2-pyhd3eb1b0_0.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/conda/user_home/anaconda3/envs/.conda_envs_dir_test b/native_locator/tests/unix/conda/user_home/anaconda3/envs/.conda_envs_dir_test new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/conda/user_home/anaconda3/envs/one/conda-meta/python-10.0.1-hdf0ec26_0_cpython.json b/native_locator/tests/unix/conda/user_home/anaconda3/envs/one/conda-meta/python-10.0.1-hdf0ec26_0_cpython.json new file mode 100644 index 000000000000..23127993ac05 --- /dev/null +++ b/native_locator/tests/unix/conda/user_home/anaconda3/envs/one/conda-meta/python-10.0.1-hdf0ec26_0_cpython.json @@ -0,0 +1 @@ +10.1.1 diff --git a/native_locator/tests/unix/conda/user_home/anaconda3/envs/one/conda-meta/python-slugify-5.0.2-pyhd3eb1b0_0.json b/native_locator/tests/unix/conda/user_home/anaconda3/envs/one/conda-meta/python-slugify-5.0.2-pyhd3eb1b0_0.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/conda/user_home/anaconda3/envs/one/python b/native_locator/tests/unix/conda/user_home/anaconda3/envs/one/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/conda/user_home/anaconda3/envs/two/python b/native_locator/tests/unix/conda/user_home/anaconda3/envs/two/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/conda_custom_install_path/user_home/.conda/environments.txt b/native_locator/tests/unix/conda_custom_install_path/user_home/.conda/environments.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/conda_custom_install_path/user_home/some_location/anaconda3/bin/conda b/native_locator/tests/unix/conda_custom_install_path/user_home/some_location/anaconda3/bin/conda new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/conda_custom_install_path/user_home/some_location/anaconda3/bin/python b/native_locator/tests/unix/conda_custom_install_path/user_home/some_location/anaconda3/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/conda_custom_install_path/user_home/some_location/anaconda3/conda-meta/conda-4.0.2-pyhd3eb1b0_0.json b/native_locator/tests/unix/conda_custom_install_path/user_home/some_location/anaconda3/conda-meta/conda-4.0.2-pyhd3eb1b0_0.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/conda_custom_install_path/user_home/some_location/anaconda3/conda-meta/python-10.0.1-hdf0ec26_0_cpython.json b/native_locator/tests/unix/conda_custom_install_path/user_home/some_location/anaconda3/conda-meta/python-10.0.1-hdf0ec26_0_cpython.json new file mode 100644 index 000000000000..23127993ac05 --- /dev/null +++ b/native_locator/tests/unix/conda_custom_install_path/user_home/some_location/anaconda3/conda-meta/python-10.0.1-hdf0ec26_0_cpython.json @@ -0,0 +1 @@ +10.1.1 diff --git a/native_locator/tests/unix/conda_custom_install_path/user_home/some_location/anaconda3/conda-meta/python-slugify-5.0.2-pyhd3eb1b0_0.json b/native_locator/tests/unix/conda_custom_install_path/user_home/some_location/anaconda3/conda-meta/python-slugify-5.0.2-pyhd3eb1b0_0.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/conda_custom_install_path/user_home/some_location/anaconda3/envs/.conda_envs_dir_test b/native_locator/tests/unix/conda_custom_install_path/user_home/some_location/anaconda3/envs/.conda_envs_dir_test new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/conda_without_envs/user_home/.conda/environments.txt b/native_locator/tests/unix/conda_without_envs/user_home/.conda/environments.txt new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/conda_without_envs/user_home/anaconda3/bin/conda b/native_locator/tests/unix/conda_without_envs/user_home/anaconda3/bin/conda new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/conda_without_envs/user_home/anaconda3/conda-meta/conda-4.0.2-pyhd3eb1b0_0.json b/native_locator/tests/unix/conda_without_envs/user_home/anaconda3/conda-meta/conda-4.0.2-pyhd3eb1b0_0.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/conda_without_envs/user_home/anaconda3/conda-meta/python-10.0.1-hdf0ec26_0_cpython.json b/native_locator/tests/unix/conda_without_envs/user_home/anaconda3/conda-meta/python-10.0.1-hdf0ec26_0_cpython.json new file mode 100644 index 000000000000..23127993ac05 --- /dev/null +++ b/native_locator/tests/unix/conda_without_envs/user_home/anaconda3/conda-meta/python-10.0.1-hdf0ec26_0_cpython.json @@ -0,0 +1 @@ +10.1.1 diff --git a/native_locator/tests/unix/conda_without_envs/user_home/anaconda3/conda-meta/python-slugify-5.0.2-pyhd3eb1b0_0.json b/native_locator/tests/unix/conda_without_envs/user_home/anaconda3/conda-meta/python-slugify-5.0.2-pyhd3eb1b0_0.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/conda_without_envs/user_home/anaconda3/envs/.conda_envs_dir_test b/native_locator/tests/unix/conda_without_envs/user_home/anaconda3/envs/.conda_envs_dir_test new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/known/user_home/python b/native_locator/tests/unix/known/user_home/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/known/user_home/python.version b/native_locator/tests/unix/known/user_home/python.version new file mode 100644 index 000000000000..4044f90867df --- /dev/null +++ b/native_locator/tests/unix/known/user_home/python.version @@ -0,0 +1 @@ +12.0.0 diff --git a/native_locator/tests/unix/pyenv/home/opt/homebrew/bin/pyenv b/native_locator/tests/unix/pyenv/home/opt/homebrew/bin/pyenv new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/3.12.1/bin/python b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/3.12.1/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/3.12.1a3/bin/python b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/3.12.1a3/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/3.13-dev/bin/python b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/3.13-dev/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/3.9.9/bin/python b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/3.9.9/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/anaconda-4.0.0/bin/envs/.conda_envs_dir_test b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/anaconda-4.0.0/bin/envs/.conda_envs_dir_test new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/anaconda-4.0.0/bin/python b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/anaconda-4.0.0/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/anaconda3-2021.04/bin/envs/.conda_envs_dir_test b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/anaconda3-2021.04/bin/envs/.conda_envs_dir_test new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/anaconda3-2021.04/bin/python b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/anaconda3-2021.04/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/mambaforge-4.10.1-4/bin/envs/.conda_envs_dir_test b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/mambaforge-4.10.1-4/bin/envs/.conda_envs_dir_test new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/mambaforge-4.10.1-4/bin/python b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/mambaforge-4.10.1-4/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/mambaforge/bin/envs/.conda_envs_dir_test b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/mambaforge/bin/envs/.conda_envs_dir_test new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/mambaforge/bin/python b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/mambaforge/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniconda-latest/bin/envs/.conda_envs_dir_test b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniconda-latest/bin/envs/.conda_envs_dir_test new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniconda-latest/bin/python b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniconda-latest/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniconda3-3.10-22.11.1-1/bin/envs/.conda_envs_dir_test b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniconda3-3.10-22.11.1-1/bin/envs/.conda_envs_dir_test new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniconda3-3.10-22.11.1-1/bin/python b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniconda3-3.10-22.11.1-1/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniconda3-3.10.1/bin/envs/.conda_envs_dir_test b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniconda3-3.10.1/bin/envs/.conda_envs_dir_test new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniconda3-3.10.1/bin/python b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniconda3-3.10.1/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniconda3-4.0.5/bin/envs/.conda_envs_dir_test b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniconda3-4.0.5/bin/envs/.conda_envs_dir_test new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniconda3-4.0.5/bin/python b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniconda3-4.0.5/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniforge3-4.11.0-1/bin/envs/.conda_envs_dir_test b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniforge3-4.11.0-1/bin/envs/.conda_envs_dir_test new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniforge3-4.11.0-1/bin/python b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/miniforge3-4.11.0-1/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/my-virtual-env/bin/python b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/my-virtual-env/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/my-virtual-env/pyvenv.cfg b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/my-virtual-env/pyvenv.cfg new file mode 100644 index 000000000000..6190a656901f --- /dev/null +++ b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/my-virtual-env/pyvenv.cfg @@ -0,0 +1,3 @@ +home = /Users/donjayamanne/.pyenv/versions/3.10.13/bin +include-system-site-packages = false +version = 3.10.13 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/nogil-3.9.10/bin/python b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/nogil-3.9.10/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/pypy3.10-7.3.14/bin/python b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/pypy3.10-7.3.14/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/pyston-2.3.5/bin/python b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/pyston-2.3.5/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/stacklets-3.7.5/bin/python b/native_locator/tests/unix/pyenv/user_home/.pyenv/versions/stacklets-3.7.5/bin/python new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/native_locator/tests/unix/pyenv_without_envs/home/opt/homebrew/bin/pyenv b/native_locator/tests/unix/pyenv_without_envs/home/opt/homebrew/bin/pyenv new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/noxfile.py b/noxfile.py index 7eb2da93cfe3..a53828df51c4 100644 --- a/noxfile.py +++ b/noxfile.py @@ -4,6 +4,7 @@ import pathlib import nox import shutil +import sys @nox.session() @@ -41,3 +42,31 @@ def install_python_libs(session: nox.Session): if pathlib.Path("./python_files/lib/temp").exists(): shutil.rmtree("./python_files/lib/temp") + + +@nox.session() +def native_build(session:nox.Session): + with session.cd("./native_locator"): + session.run("cargo", "build", "--release", "--package", "python-finder", external=True) + if not pathlib.Path(pathlib.Path.cwd() / "bin").exists(): + pathlib.Path(pathlib.Path.cwd() / "bin").mkdir() + + if not pathlib.Path(pathlib.Path.cwd() / "bin" / ".gitignore").exists(): + pathlib.Path(pathlib.Path.cwd() / "bin" / ".gitignore").write_text("*\n", encoding="utf-8") + + if sys.platform == "win32": + shutil.copy( + "./target/release/python-finder.exe", + "./bin/python-finder.exe", + ) + else: + shutil.copy( + "./target/release/python-finder", + "./bin/python-finder", + ) + + +@nox.session() +def setup_repo(session: nox.Session): + install_python_libs(session) + native_build(session) diff --git a/package.json b/package.json index 46c24ab31d01..8b7cade4784e 100644 --- a/package.json +++ b/package.json @@ -306,6 +306,16 @@ "command": "python.execSelectionInTerminal", "title": "%python.command.python.execSelectionInTerminal.title%" }, + { + "category": "Python", + "command": "python.execInREPL", + "title": "%python.command.python.execInREPL.title%" + }, + { + "category": "Python", + "command": "python.execInREPLEnter", + "title": "%python.command.python.execInREPLEnter.title%" + }, { "category": "Python", "command": "python.launchTensorBoard", @@ -437,7 +447,8 @@ "pythonDiscoveryUsingWorkers", "pythonTestAdapter", "pythonREPLSmartSend", - "pythonRecommendTensorboardExt" + "pythonRecommendTensorboardExt", + "pythonRunREPL" ], "enumDescriptions": [ "%python.experiments.All.description%", @@ -447,7 +458,9 @@ "%python.experiments.pythonDiscoveryUsingWorkers.description%", "%python.experiments.pythonTestAdapter.description%", "%python.experiments.pythonREPLSmartSend.description%", - "%python.experiments.pythonRecommendTensorboardExt.description%" + "%python.experiments.pythonRecommendTensorboardExt.description%", + "%python.experiments.pythonRunREPL.description%", + "%python.command.python.execInREPLEnter.title%" ] }, "scope": "window", @@ -465,7 +478,8 @@ "pythonTerminalEnvVarActivation", "pythonDiscoveryUsingWorkers", "pythonTestAdapter", - "pythonREPLSmartSend" + "pythonREPLSmartSend", + "pythonRunREPL" ], "enumDescriptions": [ "%python.experiments.All.description%", @@ -474,7 +488,8 @@ "%python.experiments.pythonTerminalEnvVarActivation.description%", "%python.experiments.pythonDiscoveryUsingWorkers.description%", "%python.experiments.pythonTestAdapter.description%", - "%python.experiments.pythonREPLSmartSend.description%" + "%python.experiments.pythonREPLSmartSend.description%", + "%python.experiments.pythonRunREPL.description%" ] }, "scope": "window", @@ -550,6 +565,19 @@ "experimental" ] }, + "python.locator": { + "default": "js", + "description": "%python.locator.description%", + "enum": [ + "js", + "native" + ], + "tags": [ + "experimental" + ], + "scope": "machine", + "type": "string" + }, "python.pipenvPath": { "default": "pipenv", "description": "%python.pipenvPath.description%", @@ -1156,10 +1184,10 @@ ], "menus": { "issue/reporter": [ - { - "command": "python.reportIssue" - } - ], + { + "command": "python.reportIssue" + } + ], "commandPalette": [ { "category": "Python", @@ -1241,6 +1269,12 @@ "title": "%python.command.python.execSelectionInTerminal.title%", "when": "!virtualWorkspace && shellExecutionSupported && editorLangId == python" }, + { + "category": "Python", + "command": "python.execInREPL", + "title": "%python.command.python.execInREPL.title%", + "when": "false" + }, { "category": "Python", "command": "python.launchTensorBoard", @@ -1340,6 +1374,11 @@ "command": "python.execSelectionInTerminal", "group": "Python", "when": "editorFocus && editorLangId == python && !virtualWorkspace && shellExecutionSupported" + }, + { + "command": "python.execInREPL", + "group": "Python", + "when": "editorFocus && editorLangId == python && !virtualWorkspace && shellExecutionSupported && pythonRunREPL" } ], "editor/title": [ diff --git a/package.nls.json b/package.nls.json index b88b04ab241f..aa84f31a91b2 100644 --- a/package.nls.json +++ b/package.nls.json @@ -14,6 +14,7 @@ "python.command.python.configureTests.title": "Configure Tests", "python.command.testing.rerunFailedTests.title": "Rerun Failed Tests", "python.command.python.execSelectionInTerminal.title": "Run Selection/Line in Python Terminal", + "python.command.python.execInREPL.title": "Run Selection/Line in Python REPL", "python.command.python.execSelectionInDjangoShell.title": "Run Selection/Line in Django Shell", "python.command.python.reportIssue.title": "Report Issue...", "python.command.python.enableSourceMapSupport.title": "Enable Source Map Support For Extension Debugging", @@ -44,6 +45,7 @@ "python.experiments.pythonTestAdapter.description": "Denotes the Python Test Adapter experiment.", "python.experiments.pythonREPLSmartSend.description": "Denotes the Python REPL Smart Send experiment.", "python.experiments.pythonRecommendTensorboardExt.description": "Denotes the Tensorboard Extension recommendation experiment.", + "python.experiments.pythonRunREPL.description": "Enables users to run code in interactive Python REPL.", "python.globalModuleInstallation.description": "Whether to install Python modules globally when not using an environment.", "python.languageServer.description": "Defines type of the language server.", "python.languageServer.defaultDescription": "Automatically select a language server: Pylance if installed and available, otherwise fallback to Jedi.", @@ -57,6 +59,7 @@ "python.logging.level.description": "The logging level the extension logs at, defaults to 'error'", "python.logging.level.deprecation": "This setting is deprecated. Please use command `Developer: Set Log Level...` to set logging level.", "python.missingPackage.severity.description": "Set severity of missing packages in requirements.txt or pyproject.toml", + "python.locator.description": "[Experimental] Select implementation of environment locators. This is an experimental setting while we test native environment location.", "python.pipenvPath.description": "Path to the pipenv executable to use for activation.", "python.poetryPath.description": "Path to the poetry executable.", "python.EnableREPLSmartSend.description": "Toggle Smart Send for the Python REPL. Smart Send enables sending the smallest runnable block of code to the REPL on Shift+Enter and moves the cursor accordingly.", @@ -162,4 +165,4 @@ "walkthrough.step.python.createNewNotebook.altText": "Creating a new Jupyter notebook", "walkthrough.step.python.openInteractiveWindow.altText": "Opening Python interactive window", "walkthrough.step.python.dataScienceLearnMore.altText": "Image representing our documentation page and mailing list resources." -} \ No newline at end of file +} diff --git a/python_files/python_server.py b/python_files/python_server.py new file mode 100644 index 000000000000..4d27a168bc4c --- /dev/null +++ b/python_files/python_server.py @@ -0,0 +1,167 @@ +from typing import Dict, List, Optional, Union + +import sys +import json +import contextlib +import io +import traceback +import uuid + +STDIN = sys.stdin +STDOUT = sys.stdout +STDERR = sys.stderr +USER_GLOBALS = {} + + +def send_message(msg: str): + length_msg = len(msg) + STDOUT.buffer.write(f"Content-Length: {length_msg}\r\n\r\n{msg}".encode(encoding="utf-8")) + STDOUT.buffer.flush() + + +def print_log(msg: str): + send_message(json.dumps({"jsonrpc": "2.0", "method": "log", "params": msg})) + + +def send_response(response: str, response_id: int): + send_message(json.dumps({"jsonrpc": "2.0", "id": response_id, "result": response})) + + +def send_request(params: Optional[Union[List, Dict]] = None): + request_id = uuid.uuid4().hex + if params is None: + send_message(json.dumps({"jsonrpc": "2.0", "id": request_id, "method": "input"})) + else: + send_message( + json.dumps({"jsonrpc": "2.0", "id": request_id, "method": "input", "params": params}) + ) + return request_id + + +original_input = input + + +def custom_input(prompt=""): + try: + send_request({"prompt": prompt}) + headers = get_headers() + content_length = int(headers.get("Content-Length", 0)) + + if content_length: + message_text = STDIN.read(content_length) + message_json = json.loads(message_text) + our_user_input = message_json["result"]["userInput"] + return our_user_input + except Exception: + print_log(traceback.format_exc()) + + +# Set input to our custom input +USER_GLOBALS["input"] = custom_input +input = custom_input + + +def handle_response(request_id): + while not STDIN.closed: + try: + headers = get_headers() + content_length = int(headers.get("Content-Length", 0)) + + if content_length: + message_text = STDIN.read(content_length) + message_json = json.loads(message_text) + our_user_input = message_json["result"]["userInput"] + if message_json["id"] == request_id: + send_response(our_user_input, message_json["id"]) + elif message_json["method"] == "exit": + sys.exit(0) + + except Exception: + print_log(traceback.format_exc()) + + +def exec_function(user_input): + try: + compile(user_input, "", "eval") + except SyntaxError: + return exec + return eval + + +def execute(request, user_globals): + str_output = CustomIO("", encoding="utf-8") + str_error = CustomIO("", encoding="utf-8") + + with redirect_io("stdout", str_output): + with redirect_io("stderr", str_error): + str_input = CustomIO("", encoding="utf-8", newline="\n") + with redirect_io("stdin", str_input): + exec_user_input(request["params"], user_globals) + send_response(str_output.get_value(), request["id"]) + + +def exec_user_input(user_input, user_globals): + user_input = user_input[0] if isinstance(user_input, list) else user_input + + try: + callable = exec_function(user_input) + retval = callable(user_input, user_globals) + if retval is not None: + print(retval) + except KeyboardInterrupt: + print(traceback.format_exc()) + except Exception: + print(traceback.format_exc()) + + +class CustomIO(io.TextIOWrapper): + """Custom stream object to replace stdio.""" + + def __init__(self, name, encoding="utf-8", newline=None): + self._buffer = io.BytesIO() + self._custom_name = name + super().__init__(self._buffer, encoding=encoding, newline=newline) + + def close(self): + """Provide this close method which is used by some tools.""" + # This is intentionally empty. + + def get_value(self) -> str: + """Returns value from the buffer as string.""" + self.seek(0) + return self.read() + + +@contextlib.contextmanager +def redirect_io(stream: str, new_stream): + """Redirect stdio streams to a custom stream.""" + old_stream = getattr(sys, stream) + setattr(sys, stream, new_stream) + yield + setattr(sys, stream, old_stream) + + +def get_headers(): + headers = {} + while line := STDIN.readline().strip(): + name, value = line.split(":", 1) + headers[name] = value.strip() + return headers + + +if __name__ == "__main__": + while not STDIN.closed: + try: + headers = get_headers() + content_length = int(headers.get("Content-Length", 0)) + + if content_length: + request_text = STDIN.read(content_length) + request_json = json.loads(request_text) + if request_json["method"] == "execute": + execute(request_json, USER_GLOBALS) + elif request_json["method"] == "exit": + sys.exit(0) + + except Exception: + print_log(traceback.format_exc()) diff --git a/src/client/common/application/commands.ts b/src/client/common/application/commands.ts index 30ba5d84cf5f..626321566332 100644 --- a/src/client/common/application/commands.ts +++ b/src/client/common/application/commands.ts @@ -38,6 +38,7 @@ interface ICommandNameWithoutArgumentTypeMapping { [Commands.Enable_SourceMap_Support]: []; [Commands.Exec_Selection_In_Terminal]: []; [Commands.Exec_Selection_In_Django_Shell]: []; + [Commands.Exec_In_REPL]: []; [Commands.Create_Terminal]: []; [Commands.PickLocalProcess]: []; [Commands.ClearStorage]: []; diff --git a/src/client/common/constants.ts b/src/client/common/constants.ts index 0eaade703371..ef067cb53162 100644 --- a/src/client/common/constants.ts +++ b/src/client/common/constants.ts @@ -46,6 +46,8 @@ export namespace Commands { export const Exec_In_Terminal = 'python.execInTerminal'; export const Exec_In_Terminal_Icon = 'python.execInTerminal-icon'; export const Exec_In_Separate_Terminal = 'python.execInDedicatedTerminal'; + export const Exec_In_REPL = 'python.execInREPL'; + export const Exec_In_REPL_Enter = 'python.execInREPLEnter'; export const Exec_Selection_In_Django_Shell = 'python.execSelectionInDjangoShell'; export const Exec_Selection_In_Terminal = 'python.execSelectionInTerminal'; export const GetSelectedInterpreterPath = 'python.interpreterPath'; diff --git a/src/client/common/experiments/groups.ts b/src/client/common/experiments/groups.ts index 81f157751346..543b1e27516f 100644 --- a/src/client/common/experiments/groups.ts +++ b/src/client/common/experiments/groups.ts @@ -30,3 +30,8 @@ export enum RecommendTensobardExtension { export enum CreateEnvOnPipInstallTrigger { experiment = 'pythonCreateEnvOnPipInstall', } + +// Experiment to enable running Python REPL using IW. +export enum EnableRunREPL { + experiment = 'pythonRunREPL', +} diff --git a/src/client/common/types.ts b/src/client/common/types.ts index 67fcf5c7b700..8edc76ff2bff 100644 --- a/src/client/common/types.ts +++ b/src/client/common/types.ts @@ -200,6 +200,7 @@ export interface ITerminalSettings { export interface IREPLSettings { readonly enableREPLSmartSend: boolean; + readonly enableIWREPL: boolean; } export interface IExperiments { diff --git a/src/client/common/utils/async.ts b/src/client/common/utils/async.ts index c119d8f19b06..a99db8e94562 100644 --- a/src/client/common/utils/async.ts +++ b/src/client/common/utils/async.ts @@ -155,6 +155,7 @@ export async function* chain( ): IAsyncIterableIterator { const promises = iterators.map(getNext); let numRunning = iterators.length; + while (numRunning > 0) { // Promise.race will not fail, because each promise calls getNext, // Which handles failures by wrapping each iterator in a try/catch block. diff --git a/src/client/extensionActivation.ts b/src/client/extensionActivation.ts index 543d2d0b7f49..7c582eb63239 100644 --- a/src/client/extensionActivation.ts +++ b/src/client/extensionActivation.ts @@ -3,7 +3,7 @@ 'use strict'; -import { DebugConfigurationProvider, debug, languages, window } from 'vscode'; +import { DebugConfigurationProvider, debug, languages, window, commands } from 'vscode'; import { registerTypes as activationRegisterTypes } from './activation/serviceRegistry'; import { IExtensionActivationManager } from './activation/types'; @@ -16,6 +16,7 @@ import { IFileSystem } from './common/platform/types'; import { IConfigurationService, IDisposableRegistry, + IExperimentService, IExtensions, IInterpreterPathService, ILogOutputChannel, @@ -52,6 +53,8 @@ import { initializePersistentStateForTriggers } from './common/persistentState'; import { logAndNotifyOnLegacySettings } from './logging/settingLogs'; import { DebuggerTypeName } from './debugger/constants'; import { StopWatch } from './common/utils/stopWatch'; +import { registerReplCommands } from './repl/replCommands'; +import { EnableRunREPL } from './common/experiments/groups'; export async function activateComponents( // `ext` is passed to any extra activation funcs. @@ -105,6 +108,17 @@ export function activateFeatures(ext: ExtensionState, _components: Components): interpreterService, pathUtils, ); + + // Register native REPL context menu when in experiment + const experimentService = ext.legacyIOC.serviceContainer.get(IExperimentService); + commands.executeCommand('setContext', 'pythonRunREPL', false); + if (experimentService) { + const replExperimentValue = experimentService.inExperimentSync(EnableRunREPL.experiment); + if (replExperimentValue) { + registerReplCommands(ext.disposables, interpreterService); + commands.executeCommand('setContext', 'pythonRunREPL', true); + } + } } /// ////////////////////////// diff --git a/src/client/pythonEnvironments/base/info/env.ts b/src/client/pythonEnvironments/base/info/env.ts index b77acde5333d..930a522ef1e9 100644 --- a/src/client/pythonEnvironments/base/info/env.ts +++ b/src/client/pythonEnvironments/base/info/env.ts @@ -276,14 +276,20 @@ export function areSameEnv( if (leftInfo === undefined || rightInfo === undefined) { return undefined; } - const leftFilename = leftInfo.executable!.filename; - const rightFilename = rightInfo.executable!.filename; - + if ( + (leftInfo.executable?.filename && !rightInfo.executable?.filename) || + (!leftInfo.executable?.filename && rightInfo.executable?.filename) + ) { + return false; + } if (leftInfo.id && leftInfo.id === rightInfo.id) { // In case IDs are available, use it. return true; } + const leftFilename = leftInfo.executable!.filename; + const rightFilename = rightInfo.executable!.filename; + if (getEnvID(leftFilename, leftInfo.location) === getEnvID(rightFilename, rightInfo.location)) { // Otherwise use ID function to get the ID. Note ID returned by function may itself change if executable of // an environment changes, for eg. when conda installs python into the env. So only use it as a fallback if diff --git a/src/client/pythonEnvironments/base/info/environmentInfoService.ts b/src/client/pythonEnvironments/base/info/environmentInfoService.ts index 6a981d21b6df..251834b29683 100644 --- a/src/client/pythonEnvironments/base/info/environmentInfoService.ts +++ b/src/client/pythonEnvironments/base/info/environmentInfoService.ts @@ -106,6 +106,13 @@ class EnvironmentInfoService implements IEnvironmentInfoService { } const deferred = createDeferred(); + const info = EnvironmentInfoService.getInterpreterInfo(env); + if (info !== undefined) { + this.cache.set(normCasePath(interpreterPath), deferred); + deferred.resolve(info); + return info; + } + this.cache.set(normCasePath(interpreterPath), deferred); this._getEnvironmentInfo(env, priority) .then((r) => { @@ -205,6 +212,22 @@ class EnvironmentInfoService implements IEnvironmentInfoService { } }); } + + private static getInterpreterInfo(env: PythonEnvInfo): InterpreterInformation | undefined { + if (env.version.major > -1 && env.version.minor > -1 && env.version.micro > -1 && env.location) { + return { + arch: env.arch, + executable: { + filename: env.executable.filename, + ctime: -1, + mtime: -1, + sysPrefix: env.location, + }, + version: env.version, + }; + } + return undefined; + } } function addToQueue( diff --git a/src/client/pythonEnvironments/base/locator.ts b/src/client/pythonEnvironments/base/locator.ts index 58798627678e..8524c03536c5 100644 --- a/src/client/pythonEnvironments/base/locator.ts +++ b/src/client/pythonEnvironments/base/locator.ts @@ -5,7 +5,7 @@ import { Event, Uri } from 'vscode'; import { IAsyncIterableIterator, iterEmpty } from '../../common/utils/async'; -import { PythonEnvInfo, PythonEnvKind, PythonEnvSource } from './info'; +import { PythonEnvInfo, PythonEnvKind, PythonEnvSource, PythonVersion } from './info'; import { IPythonEnvsWatcher, PythonEnvCollectionChangedEvent, @@ -145,6 +145,8 @@ export type BasicEnvInfo = { source?: PythonEnvSource[]; envPath?: string; searchLocation?: Uri; + version?: PythonVersion; + name?: string; }; /** diff --git a/src/client/pythonEnvironments/base/locators/common/nativePythonFinder.ts b/src/client/pythonEnvironments/base/locators/common/nativePythonFinder.ts new file mode 100644 index 000000000000..7d005dd7484a --- /dev/null +++ b/src/client/pythonEnvironments/base/locators/common/nativePythonFinder.ts @@ -0,0 +1,156 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { CancellationToken, Disposable, Event, EventEmitter } from 'vscode'; +import * as ch from 'child_process'; +import * as path from 'path'; +import * as rpc from 'vscode-jsonrpc/node'; +import { PassThrough } from 'stream'; +import { isWindows } from '../../../../common/platform/platformService'; +import { EXTENSION_ROOT_DIR } from '../../../../constants'; +import { traceError, traceInfo, traceLog, traceVerbose, traceWarn } from '../../../../logging'; +import { createDeferred } from '../../../../common/utils/async'; + +const NATIVE_LOCATOR = isWindows() + ? path.join(EXTENSION_ROOT_DIR, 'native_locator', 'bin', 'python-finder.exe') + : path.join(EXTENSION_ROOT_DIR, 'native_locator', 'bin', 'python-finder'); + +export interface NativeEnvInfo { + displayName?: string; + name: string; + pythonExecutablePath?: string; + category: string; + version?: string; + pythonRunCommand?: string[]; + envPath?: string; + /** + * Path to the project directory when dealing with pipenv virtual environments. + */ + projectPath?: string; +} + +export interface NativeEnvManagerInfo { + tool: string; + executablePath: string; + version?: string; +} + +export interface NativeGlobalPythonFinder extends Disposable { + startSearch(token?: CancellationToken): Promise; + onDidFindPythonEnvironment: Event; + onDidFindEnvironmentManager: Event; +} + +interface NativeLog { + level: string; + message: string; +} + +class NativeGlobalPythonFinderImpl implements NativeGlobalPythonFinder { + private readonly _onDidFindPythonEnvironment = new EventEmitter(); + + private readonly _onDidFindEnvironmentManager = new EventEmitter(); + + public readonly onDidFindPythonEnvironment = this._onDidFindPythonEnvironment.event; + + public readonly onDidFindEnvironmentManager = this._onDidFindEnvironmentManager.event; + + public startSearch(token?: CancellationToken): Promise { + const deferred = createDeferred(); + const proc = ch.spawn(NATIVE_LOCATOR, [], { env: process.env }); + const disposables: Disposable[] = []; + + // jsonrpc package cannot handle messages coming through too quicly. + // Lets handle the messages and close the stream only when + // we have got the exit event. + const readable = new PassThrough(); + proc.stdout.pipe(readable, { end: false }); + const writable = new PassThrough(); + writable.pipe(proc.stdin, { end: false }); + const disposeStreams = new Disposable(() => { + readable.end(); + writable.end(); + }); + const connection = rpc.createMessageConnection( + new rpc.StreamMessageReader(readable), + new rpc.StreamMessageWriter(writable), + ); + + disposables.push( + connection, + disposeStreams, + connection.onError((ex) => { + disposeStreams.dispose(); + traceError('Error in Native Python Finder', ex); + }), + connection.onNotification('pythonEnvironment', (data: NativeEnvInfo) => { + this._onDidFindPythonEnvironment.fire(data); + }), + connection.onNotification('envManager', (data: NativeEnvManagerInfo) => { + this._onDidFindEnvironmentManager.fire(data); + }), + connection.onNotification('exit', () => { + traceInfo('Native Python Finder exited'); + disposeStreams.dispose(); + }), + connection.onNotification('log', (data: NativeLog) => { + switch (data.level) { + case 'info': + traceInfo(`Native Python Finder: ${data.message}`); + break; + case 'warning': + traceWarn(`Native Python Finder: ${data.message}`); + break; + case 'error': + traceError(`Native Python Finder: ${data.message}`); + break; + case 'debug': + traceVerbose(`Native Python Finder: ${data.message}`); + break; + default: + traceLog(`Native Python Finder: ${data.message}`); + } + }), + connection.onClose(() => { + deferred.resolve(); + disposables.forEach((d) => d.dispose()); + }), + { + dispose: () => { + try { + if (proc.exitCode === null) { + proc.kill(); + } + } catch (err) { + traceVerbose('Error while disposing Native Python Finder', err); + } + }, + }, + ); + + if (token) { + disposables.push( + token.onCancellationRequested(() => { + deferred.resolve(); + try { + proc.kill(); + } catch (err) { + traceVerbose('Error while handling cancellation request for Native Python Finder', err); + } + }), + ); + } + + connection.listen(); + return deferred.promise; + } + + public dispose() { + this._onDidFindPythonEnvironment.dispose(); + this._onDidFindEnvironmentManager.dispose(); + } +} + +export function createNativeGlobalPythonFinder(): NativeGlobalPythonFinder { + return new NativeGlobalPythonFinderImpl(); +} diff --git a/src/client/pythonEnvironments/base/locators/composite/envsCollectionService.ts b/src/client/pythonEnvironments/base/locators/composite/envsCollectionService.ts index a54489e34633..30da89c87628 100644 --- a/src/client/pythonEnvironments/base/locators/composite/envsCollectionService.ts +++ b/src/client/pythonEnvironments/base/locators/composite/envsCollectionService.ts @@ -5,7 +5,7 @@ import { Event, EventEmitter } from 'vscode'; import '../../../../common/extensions'; import { createDeferred, Deferred } from '../../../../common/utils/async'; import { StopWatch } from '../../../../common/utils/stopWatch'; -import { traceError, traceVerbose } from '../../../../logging'; +import { traceError, traceInfo, traceVerbose } from '../../../../logging'; import { sendTelemetryEvent } from '../../../../telemetry'; import { EventName } from '../../../../telemetry/constants'; import { normalizePath } from '../../../common/externalDependencies'; @@ -107,14 +107,18 @@ export class EnvsCollectionService extends PythonEnvsWatcher { - const stopWatch = new StopWatch(); let refreshPromise = this.getRefreshPromiseForQuery(query); if (!refreshPromise) { if (options?.ifNotTriggerredAlready && this.hasRefreshFinished(query)) { // Do not trigger another refresh if a refresh has previously finished. return Promise.resolve(); } - refreshPromise = this.startRefresh(query).then(() => this.sendTelemetry(query, stopWatch)); + const stopWatch = new StopWatch(); + traceInfo(`Starting Environment refresh`); + refreshPromise = this.startRefresh(query).then(() => { + this.sendTelemetry(query, stopWatch); + traceInfo(`Environment refresh took ${stopWatch.elapsedTime} milliseconds`); + }); } return refreshPromise; } @@ -139,7 +143,7 @@ export class EnvsCollectionService extends PythonEnvsWatcher(); - + const stopWatch = new StopWatch(); if (iterator.onUpdated !== undefined) { const listener = iterator.onUpdated(async (event) => { if (isProgressEvent(event)) { @@ -147,9 +151,13 @@ export class EnvsCollectionService extends PythonEnvsWatcher { } else { executable = await conda.getInterpreterPathForEnvironment({ prefix: envPath }); } + const version = env.version ?? (executable ? await getPythonVersionFromPath(executable) : undefined); const info = buildEnvInfo({ executable, kind: PythonEnvKind.Conda, org: AnacondaCompanyName, location: envPath, source: [], - version: executable ? await getPythonVersionFromPath(executable) : undefined, + version, type: PythonEnvType.Conda, + name: env.name ?? (await conda?.getName(envPath)), }); - const name = await conda?.getName(envPath); - if (name) { - info.name = name; - } + if (env.envPath && path.basename(executable) === executable) { // For environments without python, set ID using the predicted executable path after python is installed. // Another alternative could've been to set ID of all conda environments to the environment path, as that diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/activeStateLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/activeStateLocator.ts index dc507b9c94bd..3fbdacc639a5 100644 --- a/src/client/pythonEnvironments/base/locators/lowLevel/activeStateLocator.ts +++ b/src/client/pythonEnvironments/base/locators/lowLevel/activeStateLocator.ts @@ -6,21 +6,23 @@ import { ActiveState } from '../../../common/environmentManagers/activestate'; import { PythonEnvKind } from '../../info'; import { BasicEnvInfo, IPythonEnvsIterator } from '../../locator'; -import { traceError, traceVerbose } from '../../../../logging'; +import { traceError, traceInfo, traceVerbose } from '../../../../logging'; import { LazyResourceBasedLocator } from '../common/resourceBasedLocator'; import { findInterpretersInDir } from '../../../common/commonUtils'; +import { StopWatch } from '../../../../common/utils/stopWatch'; export class ActiveStateLocator extends LazyResourceBasedLocator { public readonly providerId: string = 'activestate'; // eslint-disable-next-line class-methods-use-this public async *doIterEnvs(): IPythonEnvsIterator { + const stopWatch = new StopWatch(); const state = await ActiveState.getState(); if (state === undefined) { traceVerbose(`Couldn't locate the state binary.`); return; } - traceVerbose(`Searching for active state environments`); + traceInfo(`Searching for active state environments`); const projects = await state.getProjects(); if (projects === undefined) { traceVerbose(`Couldn't fetch State Tool projects.`); @@ -41,6 +43,6 @@ export class ActiveStateLocator extends LazyResourceBasedLocator { } } } - traceVerbose(`Finished searching for active state environments`); + traceInfo(`Finished searching for active state environments: ${stopWatch.elapsedTime} milliseconds`); } } diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/condaLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/condaLocator.ts index a58bfdd65b2c..bb48ba75b9dd 100644 --- a/src/client/pythonEnvironments/base/locators/lowLevel/condaLocator.ts +++ b/src/client/pythonEnvironments/base/locators/lowLevel/condaLocator.ts @@ -4,8 +4,9 @@ import '../../../../common/extensions'; import { PythonEnvKind } from '../../info'; import { BasicEnvInfo, IPythonEnvsIterator } from '../../locator'; import { Conda, getCondaEnvironmentsTxt } from '../../../common/environmentManagers/conda'; -import { traceError, traceVerbose } from '../../../../logging'; +import { traceError, traceInfo, traceVerbose } from '../../../../logging'; import { FSWatchingLocator } from './fsWatchingLocator'; +import { StopWatch } from '../../../../common/utils/stopWatch'; export class CondaEnvironmentLocator extends FSWatchingLocator { public readonly providerId: string = 'conda-envs'; @@ -20,6 +21,8 @@ export class CondaEnvironmentLocator extends FSWatchingLocator { // eslint-disable-next-line class-methods-use-this public async *doIterEnvs(_: unknown): IPythonEnvsIterator { + const stopWatch = new StopWatch(); + traceInfo('Searching for conda environments'); const conda = await Conda.getConda(); if (conda === undefined) { traceVerbose(`Couldn't locate the conda binary.`); @@ -38,6 +41,6 @@ export class CondaEnvironmentLocator extends FSWatchingLocator { traceError(`Failed to process conda env: ${JSON.stringify(env)}`, ex); } } - traceVerbose(`Finished searching for conda environments`); + traceInfo(`Finished searching for conda environments: ${stopWatch.elapsedTime} milliseconds`); } } diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/customVirtualEnvLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/customVirtualEnvLocator.ts index e4daeee640c9..ae74d2f3e189 100644 --- a/src/client/pythonEnvironments/base/locators/lowLevel/customVirtualEnvLocator.ts +++ b/src/client/pythonEnvironments/base/locators/lowLevel/customVirtualEnvLocator.ts @@ -23,7 +23,8 @@ import { } from '../../../common/environmentManagers/simplevirtualenvs'; import '../../../../common/extensions'; import { asyncFilter } from '../../../../common/utils/arrayUtils'; -import { traceError, traceVerbose } from '../../../../logging'; +import { traceError, traceInfo, traceVerbose } from '../../../../logging'; +import { StopWatch } from '../../../../common/utils/stopWatch'; /** * Default number of levels of sub-directories to recurse when looking for interpreters. */ @@ -99,6 +100,8 @@ export class CustomVirtualEnvironmentLocator extends FSWatchingLocator { // eslint-disable-next-line class-methods-use-this protected doIterEnvs(): IPythonEnvsIterator { async function* iterator() { + const stopWatch = new StopWatch(); + traceInfo('Searching for custom virtual environments'); const envRootDirs = await getCustomVirtualEnvDirs(); const envGenerators = envRootDirs.map((envRootDir) => { async function* generator() { @@ -132,7 +135,7 @@ export class CustomVirtualEnvironmentLocator extends FSWatchingLocator { }); yield* iterable(chain(envGenerators)); - traceVerbose(`Finished searching for custom virtual envs`); + traceInfo(`Finished searching for custom virtual envs: ${stopWatch.elapsedTime} milliseconds`); } return iterator(); diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/globalVirtualEnvronmentLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/globalVirtualEnvronmentLocator.ts index cc623be8392d..3964a6ceb893 100644 --- a/src/client/pythonEnvironments/base/locators/lowLevel/globalVirtualEnvronmentLocator.ts +++ b/src/client/pythonEnvironments/base/locators/lowLevel/globalVirtualEnvronmentLocator.ts @@ -19,7 +19,8 @@ import { } from '../../../common/environmentManagers/simplevirtualenvs'; import '../../../../common/extensions'; import { asyncFilter } from '../../../../common/utils/arrayUtils'; -import { traceError, traceVerbose } from '../../../../logging'; +import { traceError, traceInfo, traceVerbose } from '../../../../logging'; +import { StopWatch } from '../../../../common/utils/stopWatch'; const DEFAULT_SEARCH_DEPTH = 2; /** @@ -118,6 +119,8 @@ export class GlobalVirtualEnvironmentLocator extends FSWatchingLocator { const searchDepth = this.searchDepth ?? DEFAULT_SEARCH_DEPTH; async function* iterator() { + const stopWatch = new StopWatch(); + traceInfo('Searching for global virtual environments'); const envRootDirs = await getGlobalVirtualEnvDirs(); const envGenerators = envRootDirs.map((envRootDir) => { async function* generator() { @@ -152,7 +155,7 @@ export class GlobalVirtualEnvironmentLocator extends FSWatchingLocator { }); yield* iterable(chain(envGenerators)); - traceVerbose(`Finished searching for global virtual envs`); + traceInfo(`Finished searching for global virtual envs: ${stopWatch.elapsedTime} milliseconds`); } return iterator(); diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/microsoftStoreLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/microsoftStoreLocator.ts index 7adeeae89858..60528bd939aa 100644 --- a/src/client/pythonEnvironments/base/locators/lowLevel/microsoftStoreLocator.ts +++ b/src/client/pythonEnvironments/base/locators/lowLevel/microsoftStoreLocator.ts @@ -12,7 +12,8 @@ import { isStorePythonInstalled, getMicrosoftStoreAppsRoot, } from '../../../common/environmentManagers/microsoftStoreEnv'; -import { traceVerbose } from '../../../../logging'; +import { traceInfo } from '../../../../logging'; +import { StopWatch } from '../../../../common/utils/stopWatch'; /** * This is a glob pattern which matches following file names: @@ -87,13 +88,14 @@ export class MicrosoftStoreLocator extends FSWatchingLocator { protected doIterEnvs(): IPythonEnvsIterator { const iterator = async function* (kind: PythonEnvKind) { - traceVerbose('Searching for windows store envs'); + const stopWatch = new StopWatch(); + traceInfo('Searching for windows store envs'); const exes = await getMicrosoftStorePythonExes(); yield* exes.map(async (executablePath: string) => ({ kind, executablePath, })); - traceVerbose(`Finished searching for windows store envs`); + traceInfo(`Finished searching for windows store envs: ${stopWatch.elapsedTime} milliseconds`); }; return iterator(this.kind); } diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/nativeLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/nativeLocator.ts new file mode 100644 index 000000000000..4e9d372520f6 --- /dev/null +++ b/src/client/pythonEnvironments/base/locators/lowLevel/nativeLocator.ts @@ -0,0 +1,153 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +import { Disposable, Event, EventEmitter } from 'vscode'; +import { IDisposable } from '../../../../common/types'; +import { ILocator, BasicEnvInfo, IPythonEnvsIterator } from '../../locator'; +import { PythonEnvsChangedEvent } from '../../watcher'; +import { PythonEnvKind, PythonVersion } from '../../info'; +import { Conda } from '../../../common/environmentManagers/conda'; +import { traceError, traceInfo } from '../../../../logging'; +import type { KnownEnvironmentTools } from '../../../../api/types'; +import { setPyEnvBinary } from '../../../common/environmentManagers/pyenv'; +import { + NativeEnvInfo, + NativeEnvManagerInfo, + NativeGlobalPythonFinder, + createNativeGlobalPythonFinder, +} from '../common/nativePythonFinder'; +import { disposeAll } from '../../../../common/utils/resourceLifecycle'; +import { StopWatch } from '../../../../common/utils/stopWatch'; + +function categoryToKind(category: string): PythonEnvKind { + switch (category.toLowerCase()) { + case 'conda': + return PythonEnvKind.Conda; + case 'system': + case 'homebrew': + case 'windowsregistry': + return PythonEnvKind.System; + case 'pyenv': + return PythonEnvKind.Pyenv; + case 'pipenv': + return PythonEnvKind.Pipenv; + case 'pyenvvirtualenv': + return PythonEnvKind.VirtualEnv; + case 'virtualenvwrapper': + return PythonEnvKind.VirtualEnvWrapper; + case 'windowsstore': + return PythonEnvKind.MicrosoftStore; + default: { + traceError(`Unknown Python Environment category '${category}' from Native Locator.`); + return PythonEnvKind.Unknown; + } + } +} + +function toolToKnownEnvironmentTool(tool: string): KnownEnvironmentTools { + switch (tool.toLowerCase()) { + case 'conda': + return 'Conda'; + case 'pyenv': + return 'Pyenv'; + default: { + traceError(`Unknown Python Tool '${tool}' from Native Locator.`); + return 'Unknown'; + } + } +} + +function parseVersion(version?: string): PythonVersion | undefined { + if (!version) { + return undefined; + } + + try { + const [major, minor, micro] = version.split('.').map((v) => parseInt(v, 10)); + return { + major, + minor, + micro, + sysVersion: version, + }; + } catch { + return undefined; + } +} + +export class NativeLocator implements ILocator, IDisposable { + public readonly providerId: string = 'native-locator'; + + private readonly onChangedEmitter = new EventEmitter(); + + private readonly disposables: IDisposable[] = []; + + private readonly finder: NativeGlobalPythonFinder; + + constructor() { + this.onChanged = this.onChangedEmitter.event; + this.finder = createNativeGlobalPythonFinder(); + this.disposables.push(this.onChangedEmitter, this.finder); + } + + public readonly onChanged: Event; + + public async dispose(): Promise { + this.disposables.forEach((d) => d.dispose()); + return Promise.resolve(); + } + + public iterEnvs(): IPythonEnvsIterator { + const stopWatch = new StopWatch(); + traceInfo('Searching for Python environments using Native Locator'); + const promise = this.finder.startSearch(); + const envs: BasicEnvInfo[] = []; + const disposables: IDisposable[] = []; + const disposable = new Disposable(() => disposeAll(disposables)); + this.disposables.push(disposable); + promise.finally(() => disposable.dispose()); + disposables.push( + this.finder.onDidFindPythonEnvironment((data: NativeEnvInfo) => { + // TODO: What if executable is undefined? + if (data.pythonExecutablePath) { + envs.push({ + kind: categoryToKind(data.category), + executablePath: data.pythonExecutablePath, + envPath: data.envPath, + version: parseVersion(data.version), + name: data.name === '' ? undefined : data.name, + }); + } + }), + this.finder.onDidFindEnvironmentManager((data: NativeEnvManagerInfo) => { + switch (toolToKnownEnvironmentTool(data.tool)) { + case 'Conda': { + Conda.setConda(data.executablePath); + break; + } + case 'Pyenv': { + setPyEnvBinary(data.executablePath); + break; + } + default: { + break; + } + } + }), + ); + + const iterator = async function* (): IPythonEnvsIterator { + // When this promise is complete, we know that the search is complete. + await promise; + traceInfo( + `Finished searching for Python environments using Native Locator: ${stopWatch.elapsedTime} milliseconds`, + ); + yield* envs; + traceInfo( + `Finished yielding Python environments using Native Locator: ${stopWatch.elapsedTime} milliseconds`, + ); + }; + + return iterator(); + } +} diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/posixKnownPathsLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/posixKnownPathsLocator.ts index 2d7ebc2af111..daca4b860907 100644 --- a/src/client/pythonEnvironments/base/locators/lowLevel/posixKnownPathsLocator.ts +++ b/src/client/pythonEnvironments/base/locators/lowLevel/posixKnownPathsLocator.ts @@ -9,7 +9,8 @@ import { commonPosixBinPaths, getPythonBinFromPosixPaths } from '../../../common import { isPyenvShimDir } from '../../../common/environmentManagers/pyenv'; import { getOSType, OSType } from '../../../../common/utils/platform'; import { isMacDefaultPythonPath } from '../../../common/environmentManagers/macDefault'; -import { traceError, traceVerbose } from '../../../../logging'; +import { traceError, traceInfo, traceVerbose } from '../../../../logging'; +import { StopWatch } from '../../../../common/utils/stopWatch'; export class PosixKnownPathsLocator extends Locator { public readonly providerId = 'posixKnownPaths'; @@ -26,7 +27,8 @@ export class PosixKnownPathsLocator extends Locator { } const iterator = async function* (kind: PythonEnvKind) { - traceVerbose('Searching for interpreters in posix paths locator'); + const stopWatch = new StopWatch(); + traceInfo('Searching for interpreters in posix paths locator'); try { // Filter out pyenv shims. They are not actual python binaries, they are used to launch // the binaries specified in .python-version file in the cwd. We should not be reporting @@ -50,7 +52,9 @@ export class PosixKnownPathsLocator extends Locator { } catch (ex) { traceError('Failed to process posix paths', ex); } - traceVerbose('Finished searching for interpreters in posix paths locator'); + traceInfo( + `Finished searching for interpreters in posix paths locator: ${stopWatch.elapsedTime} milliseconds`, + ); }; return iterator(this.kind); } diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/pyenvLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/pyenvLocator.ts index 4fd1891a179c..e97b69c6b882 100644 --- a/src/client/pythonEnvironments/base/locators/lowLevel/pyenvLocator.ts +++ b/src/client/pythonEnvironments/base/locators/lowLevel/pyenvLocator.ts @@ -7,7 +7,8 @@ import { FSWatchingLocator } from './fsWatchingLocator'; import { getInterpreterPathFromDir } from '../../../common/commonUtils'; import { getSubDirs } from '../../../common/externalDependencies'; import { getPyenvVersionsDir } from '../../../common/environmentManagers/pyenv'; -import { traceError, traceVerbose } from '../../../../logging'; +import { traceError, traceInfo } from '../../../../logging'; +import { StopWatch } from '../../../../common/utils/stopWatch'; /** * Gets all the pyenv environments. @@ -16,25 +17,31 @@ import { traceError, traceVerbose } from '../../../../logging'; * all the environments (global or virtual) in that directory. */ async function* getPyenvEnvironments(): AsyncIterableIterator { - traceVerbose('Searching for pyenv environments'); - const pyenvVersionDir = getPyenvVersionsDir(); + const stopWatch = new StopWatch(); + traceInfo('Searching for pyenv environments'); + try { + const pyenvVersionDir = getPyenvVersionsDir(); - const subDirs = getSubDirs(pyenvVersionDir, { resolveSymlinks: true }); - for await (const subDirPath of subDirs) { - const interpreterPath = await getInterpreterPathFromDir(subDirPath); + const subDirs = getSubDirs(pyenvVersionDir, { resolveSymlinks: true }); + for await (const subDirPath of subDirs) { + const interpreterPath = await getInterpreterPathFromDir(subDirPath); - if (interpreterPath) { - try { - yield { - kind: PythonEnvKind.Pyenv, - executablePath: interpreterPath, - }; - } catch (ex) { - traceError(`Failed to process environment: ${interpreterPath}`, ex); + if (interpreterPath) { + try { + yield { + kind: PythonEnvKind.Pyenv, + executablePath: interpreterPath, + }; + } catch (ex) { + traceError(`Failed to process environment: ${interpreterPath}`, ex); + } } } + } catch (ex) { + // This is expected when pyenv is not installed + traceInfo(`pyenv is not installed`); } - traceVerbose('Finished searching for pyenv environments'); + traceInfo(`Finished searching for pyenv environments: ${stopWatch.elapsedTime} milliseconds`); } export class PyenvLocator extends FSWatchingLocator { diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts index d158d1da156c..440d075b4071 100644 --- a/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts +++ b/src/client/pythonEnvironments/base/locators/lowLevel/windowsKnownPathsLocator.ts @@ -16,10 +16,11 @@ import { Locators } from '../../locators'; import { getEnvs } from '../../locatorUtils'; import { PythonEnvsChangedEvent } from '../../watcher'; import { DirFilesLocator } from './filesLocator'; -import { traceVerbose } from '../../../../logging'; +import { traceInfo } from '../../../../logging'; import { inExperiment, pathExists } from '../../../common/externalDependencies'; import { DiscoveryUsingWorkers } from '../../../../common/experiments/groups'; import { iterPythonExecutablesInDir, looksLikeBasicGlobalPython } from '../../../common/commonUtils'; +import { StopWatch } from '../../../../common/utils/stopWatch'; /** * A locator for Windows locators found under the $PATH env var. @@ -68,11 +69,12 @@ export class WindowsPathEnvVarLocator implements ILocator, IDispos // are valid executables. That is left to callers (e.g. composite // locators). async function* iterator(it: IPythonEnvsIterator) { - traceVerbose(`Searching windows known paths locator`); + const stopWatch = new StopWatch(); + traceInfo(`Searching windows known paths locator`); for await (const env of it) { yield env; } - traceVerbose(`Finished searching windows known paths locator`); + traceInfo(`Finished searching windows known paths locator: ${stopWatch.elapsedTime} milliseconds`); } return iterator(this.locators.iterEnvs(query)); } diff --git a/src/client/pythonEnvironments/base/locators/lowLevel/windowsRegistryLocator.ts b/src/client/pythonEnvironments/base/locators/lowLevel/windowsRegistryLocator.ts index a574116f1854..1447c2a90767 100644 --- a/src/client/pythonEnvironments/base/locators/lowLevel/windowsRegistryLocator.ts +++ b/src/client/pythonEnvironments/base/locators/lowLevel/windowsRegistryLocator.ts @@ -6,11 +6,12 @@ import { PythonEnvKind, PythonEnvSource } from '../../info'; import { BasicEnvInfo, IPythonEnvsIterator, Locator, PythonLocatorQuery, IEmitter } from '../../locator'; import { getRegistryInterpreters } from '../../../common/windowsUtils'; -import { traceError, traceVerbose } from '../../../../logging'; +import { traceError, traceInfo } from '../../../../logging'; import { isMicrosoftStoreDir } from '../../../common/environmentManagers/microsoftStoreEnv'; import { PythonEnvsChangedEvent } from '../../watcher'; import { DiscoveryUsingWorkers } from '../../../../common/experiments/groups'; import { inExperiment } from '../../../common/externalDependencies'; +import { StopWatch } from '../../../../common/utils/stopWatch'; export const WINDOWS_REG_PROVIDER_ID = 'windows-registry'; @@ -42,13 +43,15 @@ async function* iterateEnvsLazily(changed: IEmitter): IP } async function loadAllEnvs(changed: IEmitter) { - traceVerbose('Searching for windows registry interpreters'); - await getRegistryInterpreters(); + const stopWatch = new StopWatch(); + traceInfo('Searching for windows registry interpreters'); changed.fire({ providerId: WINDOWS_REG_PROVIDER_ID }); - traceVerbose('Finished searching for windows registry interpreters'); + traceInfo(`Finished searching for windows registry interpreters: ${stopWatch.elapsedTime} milliseconds`); } async function* iterateEnvs(): IPythonEnvsIterator { + const stopWatch = new StopWatch(); + traceInfo('Searching for windows registry interpreters'); const interpreters = await getRegistryInterpreters(); // Value should already be loaded at this point, so this returns immediately. for (const interpreter of interpreters) { try { @@ -68,5 +71,5 @@ async function* iterateEnvs(): IPythonEnvsIterator { traceError(`Failed to process environment: ${interpreter}`, ex); } } - traceVerbose('Finished searching for windows registry interpreters'); + traceInfo(`Finished searching for windows registry interpreters: ${stopWatch.elapsedTime} milliseconds`); } diff --git a/src/client/pythonEnvironments/common/environmentManagers/conda.ts b/src/client/pythonEnvironments/common/environmentManagers/conda.ts index 2ec5844f4708..9f107a737dc3 100644 --- a/src/client/pythonEnvironments/common/environmentManagers/conda.ts +++ b/src/client/pythonEnvironments/common/environmentManagers/conda.ts @@ -287,6 +287,10 @@ export class Conda { return Conda.condaPromise.get(shellPath); } + public static setConda(condaPath: string): void { + Conda.condaPromise.set(undefined, Promise.resolve(new Conda(condaPath))); + } + /** * Locates the preferred "conda" utility on this system by considering user settings, * binaries on PATH, Python interpreters in the registry, and known install locations. diff --git a/src/client/pythonEnvironments/common/environmentManagers/pyenv.ts b/src/client/pythonEnvironments/common/environmentManagers/pyenv.ts index 3d6c276cc868..8556e6f19f90 100644 --- a/src/client/pythonEnvironments/common/environmentManagers/pyenv.ts +++ b/src/client/pythonEnvironments/common/environmentManagers/pyenv.ts @@ -21,7 +21,17 @@ export function getPyenvDir(): string { return pyenvDir; } +let pyenvBinary: string | undefined; + +export function setPyEnvBinary(pyenvBin: string): void { + pyenvBinary = pyenvBin; +} + async function getPyenvBinary(): Promise { + if (pyenvBinary && (await pathExists(pyenvBinary))) { + return pyenvBinary; + } + const pyenvDir = getPyenvDir(); const pyenvBin = path.join(pyenvDir, 'bin', 'pyenv'); if (await pathExists(pyenvBin)) { diff --git a/src/client/pythonEnvironments/index.ts b/src/client/pythonEnvironments/index.ts index d3f6166295d9..9406f890c6da 100644 --- a/src/client/pythonEnvironments/index.ts +++ b/src/client/pythonEnvironments/index.ts @@ -39,6 +39,8 @@ import { IDisposable } from '../common/types'; import { traceError } from '../logging'; import { ActiveStateLocator } from './base/locators/lowLevel/activeStateLocator'; import { CustomWorkspaceLocator } from './base/locators/lowLevel/customWorkspaceLocator'; +import { NativeLocator } from './base/locators/lowLevel/nativeLocator'; +import { getConfiguration } from '../common/vscodeApis/workspaceApis'; const PYTHON_ENV_INFO_CACHE_KEY = 'PYTHON_ENV_INFO_CACHEv2'; @@ -136,29 +138,38 @@ async function createLocator( return caching; } +function useNativeLocator(): boolean { + const config = getConfiguration('python'); + return config.get('locator', 'js') === 'native'; +} + function createNonWorkspaceLocators(ext: ExtensionState): ILocator[] { const locators: (ILocator & Partial)[] = []; - locators.push( - // OS-independent locators go here. - new PyenvLocator(), - new CondaEnvironmentLocator(), - new ActiveStateLocator(), - new GlobalVirtualEnvironmentLocator(), - new CustomVirtualEnvironmentLocator(), - ); - - if (getOSType() === OSType.Windows) { - locators.push( - // Windows specific locators go here. - new WindowsRegistryLocator(), - new MicrosoftStoreLocator(), - new WindowsPathEnvVarLocator(), - ); + if (useNativeLocator()) { + locators.push(new NativeLocator()); } else { locators.push( - // Linux/Mac locators go here. - new PosixKnownPathsLocator(), + // OS-independent locators go here. + new PyenvLocator(), + new CondaEnvironmentLocator(), + new ActiveStateLocator(), + new GlobalVirtualEnvironmentLocator(), + new CustomVirtualEnvironmentLocator(), ); + + if (getOSType() === OSType.Windows) { + locators.push( + // Windows specific locators go here. + new WindowsRegistryLocator(), + new MicrosoftStoreLocator(), + new WindowsPathEnvVarLocator(), + ); + } else { + locators.push( + // Linux/Mac locators go here. + new PosixKnownPathsLocator(), + ); + } } const disposables = locators.filter((d) => d.dispose !== undefined) as IDisposable[]; diff --git a/src/client/repl/pythonServer.ts b/src/client/repl/pythonServer.ts new file mode 100644 index 000000000000..e25ba3a25092 --- /dev/null +++ b/src/client/repl/pythonServer.ts @@ -0,0 +1,81 @@ +import * as path from 'path'; +import * as ch from 'child_process'; +import * as rpc from 'vscode-jsonrpc/node'; +import { Disposable, window } from 'vscode'; +import { EXTENSION_ROOT_DIR } from '../constants'; +import { traceError, traceLog } from '../logging'; + +const SERVER_PATH = path.join(EXTENSION_ROOT_DIR, 'python_files', 'python_server.py'); + +export interface PythonServer extends Disposable { + execute(code: string): Promise; + interrupt(): void; + input(): void; +} + +class PythonServerImpl implements Disposable { + constructor(private connection: rpc.MessageConnection, private pythonServer: ch.ChildProcess) { + this.initialize(); + this.input(); + } + + private initialize(): void { + this.connection.onNotification('log', (message: string) => { + console.log('Log:', message); + }); + this.connection.listen(); + } + + // Register input handler + public input(): void { + // Register input request handler + this.connection.onRequest('input', async (request) => { + // Ask for user input via popup quick input, send it back to Python + let userPrompt = 'Enter your input here: '; + if (request && request.prompt) { + userPrompt = request.prompt; + } + const input = await window.showInputBox({ + title: 'Input Request', + prompt: userPrompt, + ignoreFocusOut: true, + }); + return { userInput: input }; + }); + } + + public execute(code: string): Promise { + return this.connection.sendRequest('execute', code); + } + + public interrupt(): void { + if (this.pythonServer.kill('SIGINT')) { + traceLog('Python server interrupted'); + } + } + + public dispose(): void { + this.connection.sendNotification('exit'); + this.connection.dispose(); + } +} + +export function createPythonServer(interpreter: string[]): PythonServer { + const pythonServer = ch.spawn(interpreter[0], [...interpreter.slice(1), SERVER_PATH]); + + pythonServer.stderr.on('data', (data) => { + traceError(data.toString()); + }); + pythonServer.on('exit', (code) => { + traceError(`Python server exited with code ${code}`); + }); + pythonServer.on('error', (err) => { + traceError(err); + }); + const connection = rpc.createMessageConnection( + new rpc.StreamMessageReader(pythonServer.stdout), + new rpc.StreamMessageWriter(pythonServer.stdin), + ); + + return new PythonServerImpl(connection, pythonServer); +} diff --git a/src/client/repl/replCommands.ts b/src/client/repl/replCommands.ts new file mode 100644 index 000000000000..3cb2e80a2614 --- /dev/null +++ b/src/client/repl/replCommands.ts @@ -0,0 +1,106 @@ +import { + commands, + NotebookController, + Uri, + workspace, + window, + NotebookControllerAffinity, + ViewColumn, + NotebookEdit, + NotebookCellData, + NotebookCellKind, + WorkspaceEdit, + NotebookEditor, + TextEditor, +} from 'vscode'; +import { Disposable } from 'vscode-jsonrpc'; +import { Commands, PVSC_EXTENSION_ID } from '../common/constants'; +import { noop } from '../common/utils/misc'; +import { IInterpreterService } from '../interpreter/contracts'; +import { getMultiLineSelectionText, getSingleLineSelectionText } from '../terminals/codeExecution/helper'; +import { createReplController } from './replController'; + +let notebookController: NotebookController | undefined; +let notebookEditor: NotebookEditor | undefined; +// TODO: figure out way to put markdown telling user kernel has been dead and need to pick again. + +async function getSelectedTextToExecute(textEditor: TextEditor): Promise { + if (!textEditor) { + return undefined; + } + + const { selection } = textEditor; + let code: string; + + if (selection.isEmpty) { + code = textEditor.document.lineAt(selection.start.line).text; + } else if (selection.isSingleLine) { + code = getSingleLineSelectionText(textEditor); + } else { + code = getMultiLineSelectionText(textEditor); + } + + return code; +} + +export async function registerReplCommands( + disposables: Disposable[], + interpreterService: IInterpreterService, +): Promise { + disposables.push( + commands.registerCommand(Commands.Exec_In_REPL, async (uri: Uri) => { + const interpreter = await interpreterService.getActiveInterpreter(uri); + if (!interpreter) { + commands.executeCommand(Commands.TriggerEnvironmentSelection, uri).then(noop, noop); + return; + } + if (interpreter) { + const interpreterPath = interpreter.path; + + if (!notebookController) { + notebookController = createReplController(interpreterPath); + } + const activeEditor = window.activeTextEditor as TextEditor; + + const code = await getSelectedTextToExecute(activeEditor); + const ourResource = Uri.from({ scheme: 'untitled', path: 'repl.interactive' }); + + const notebookDocument = await workspace.openNotebookDocument(ourResource); + // commands.executeCommand('_interactive.open'); command to open interactive window so intellisense is registered. + + // We want to keep notebookEditor, whenever we want to run. + // Find interactive window, or open it. + if (!notebookEditor) { + notebookEditor = await window.showNotebookDocument(notebookDocument, { + viewColumn: ViewColumn.Beside, + }); + } + + notebookController!.updateNotebookAffinity(notebookDocument, NotebookControllerAffinity.Default); + + // Auto-Select Python REPL Kernel + await commands.executeCommand('notebook.selectKernel', { + notebookEditor, + id: notebookController?.id, + extension: PVSC_EXTENSION_ID, + }); + + const notebookCellData = new NotebookCellData(NotebookCellKind.Code, code as string, 'python'); + const { cellCount } = notebookDocument; + // Add new cell to interactive window document + const notebookEdit = NotebookEdit.insertCells(cellCount, [notebookCellData]); + const workspaceEdit = new WorkspaceEdit(); + workspaceEdit.set(notebookDocument.uri, [notebookEdit]); + await workspace.applyEdit(workspaceEdit); + + // Execute the cell + commands.executeCommand('notebook.cell.execute', { + ranges: [{ start: cellCount, end: cellCount + 1 }], + document: ourResource, + }); + } + }), + ); +} + +// Register replEnterCommand diff --git a/src/client/repl/replController.ts b/src/client/repl/replController.ts new file mode 100644 index 000000000000..f7ee7e6d486c --- /dev/null +++ b/src/client/repl/replController.ts @@ -0,0 +1,43 @@ +import * as vscode from 'vscode'; +import { createPythonServer } from './pythonServer'; + +export function createReplController(interpreterPath: string): vscode.NotebookController { + const server = createPythonServer([interpreterPath]); + const controller = vscode.notebooks.createNotebookController('pythonREPL', 'interactive', 'Python REPL'); + controller.supportedLanguages = ['python']; + controller.supportsExecutionOrder = true; + + controller.description = 'Python REPL'; + + controller.interruptHandler = async () => { + server.interrupt(); + }; + + controller.executeHandler = async (cells) => { + for (const cell of cells) { + const exec = controller.createNotebookCellExecution(cell); + exec.start(Date.now()); + try { + const result = await server.execute(cell.document.getText()); + + exec.replaceOutput([ + new vscode.NotebookCellOutput([vscode.NotebookCellOutputItem.text(result, 'text/plain')]), + ]); + exec.end(true); + } catch (err) { + const error = err as Error; + exec.replaceOutput([ + new vscode.NotebookCellOutput([ + vscode.NotebookCellOutputItem.error({ + name: error.name, + message: error.message, + stack: error.stack, + }), + ]), + ]); + exec.end(false); + } + } + }; + return controller; +} diff --git a/src/client/terminals/codeExecution/helper.ts b/src/client/terminals/codeExecution/helper.ts index 880da969d690..ff1c4f218f8d 100644 --- a/src/client/terminals/codeExecution/helper.ts +++ b/src/client/terminals/codeExecution/helper.ts @@ -200,7 +200,7 @@ export class CodeExecutionHelper implements ICodeExecutionHelper { } } -function getSingleLineSelectionText(textEditor: TextEditor): string { +export function getSingleLineSelectionText(textEditor: TextEditor): string { const { selection } = textEditor; const selectionRange = new Range(selection.start, selection.end); const selectionText = textEditor.document.getText(selectionRange); @@ -227,7 +227,7 @@ function getSingleLineSelectionText(textEditor: TextEditor): string { return selectionText; } -function getMultiLineSelectionText(textEditor: TextEditor): string { +export function getMultiLineSelectionText(textEditor: TextEditor): string { const { selection } = textEditor; const selectionRange = new Range(selection.start, selection.end); const selectionText = textEditor.document.getText(selectionRange); diff --git a/src/test/terminals/codeExecution/helper.test.ts b/src/test/terminals/codeExecution/helper.test.ts index 2ea00e77c925..9098455c968e 100644 --- a/src/test/terminals/codeExecution/helper.test.ts +++ b/src/test/terminals/codeExecution/helper.test.ts @@ -110,7 +110,9 @@ suite('Terminal - Code Execution Helper', () => { .setup((c) => c.get(TypeMoq.It.isValue(IActiveResourceService))) .returns(() => activeResourceService.object); activeResourceService.setup((a) => a.getActiveResource()).returns(() => resource); - pythonSettings.setup((s) => s.REPL).returns(() => ({ enableREPLSmartSend: false, REPLSmartSend: false })); + pythonSettings + .setup((s) => s.REPL) + .returns(() => ({ enableREPLSmartSend: false, REPLSmartSend: false, enableIWREPL: false })); configurationService.setup((x) => x.getSettings(TypeMoq.It.isAny())).returns(() => pythonSettings.object); configurationService .setup((c) => c.getSettings(TypeMoq.It.isAny())) diff --git a/src/test/terminals/codeExecution/smartSend.test.ts b/src/test/terminals/codeExecution/smartSend.test.ts index f93df2ac11ed..ba5101332bf8 100644 --- a/src/test/terminals/codeExecution/smartSend.test.ts +++ b/src/test/terminals/codeExecution/smartSend.test.ts @@ -107,7 +107,9 @@ suite('REPL - Smart Send', () => { .returns(() => activeResourceService.object); activeResourceService.setup((a) => a.getActiveResource()).returns(() => resource); - pythonSettings.setup((s) => s.REPL).returns(() => ({ enableREPLSmartSend: true, REPLSmartSend: true })); + pythonSettings + .setup((s) => s.REPL) + .returns(() => ({ enableREPLSmartSend: true, REPLSmartSend: true, enableIWREPL: false })); configurationService.setup((x) => x.getSettings(TypeMoq.It.isAny())).returns(() => pythonSettings.object);