diff --git a/native_locator/src/common_python.rs b/native_locator/src/common_python.rs index 67588ee6aeb6..5549ca454acf 100644 --- a/native_locator/src/common_python.rs +++ b/native_locator/src/common_python.rs @@ -1,27 +1,20 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +use log::warn; use regex::Regex; use crate::known::Environment; use crate::locator::{Locator, LocatorResult}; use crate::messaging::PythonEnvironment; use crate::utils::{ - self, find_python_binary_path, get_version_from_header_files, is_symlinked_python_executable, - PythonEnv, + self, find_all_python_binaries_in_path, get_shortest_python_executable, + get_version_from_header_files, is_symlinked_python_executable, PythonEnv, }; -use std::env; +use std::cell::RefCell; +use std::collections::HashMap; use std::path::{Path, PathBuf}; -fn get_env_path(python_executable_path: &PathBuf) -> Option { - let parent = python_executable_path.parent()?; - if parent.file_name()? == "Scripts" { - return Some(parent.parent()?.to_path_buf()); - } else { - return Some(parent.to_path_buf()); - } -} - pub struct PythonOnPath<'a> { pub environment: &'a dyn Environment, } @@ -44,7 +37,6 @@ impl Locator for PythonOnPath<'_> { python_run_command: Some(vec![exe.clone().to_str().unwrap().to_string()]), ..Default::default() }; - if let Some(symlink) = is_symlinked_python_executable(&exe) { env.symlinks = Some(vec![symlink.clone(), exe.clone()]); // Getting version this way is more accurate than the above regex. @@ -65,14 +57,16 @@ impl Locator for PythonOnPath<'_> { env.version = Some(version); } } + + if let Some(env_path) = symlink.ancestors().nth(2) { + env.env_path = Some(env_path.to_path_buf()); + } } } Some(env) } fn find(&mut self) -> Option { - let paths = self.environment.get_env_var("PATH".to_string())?; - // Exclude files from this folder, as they would have been discovered elsewhere (widows_store) // Also the exe is merely a pointer to another file. let home = self.environment.get_user_home()?; @@ -81,27 +75,95 @@ impl Locator for PythonOnPath<'_> { .join("Local") .join("Microsoft") .join("WindowsApps"); - let mut environments: Vec = vec![]; - env::split_paths(&paths) + let mut python_executables = self + .environment + .get_know_global_search_locations() + .into_iter() .filter(|p| !p.starts_with(apps_path.clone())) // Paths like /Library/Frameworks/Python.framework/Versions/3.10/bin can end up in the current PATH variable. // Hence do not just look for files in a bin directory of the path. - .map(|p| find_python_binary_path(&p)) - .filter(Option::is_some) - .map(Option::unwrap) - .for_each(|full_path| { - let version = utils::get_version(&full_path); - let env_path = get_env_path(&full_path); - if let Some(env) = self.resolve(&PythonEnv::new(full_path, env_path, version)) { - environments.push(env); + .map(|p| find_all_python_binaries_in_path(&p)) + .flatten() + .collect::>(); + + // The python executables can contain files like + // /usr/local/bin/python3.10 + // /usr/local/bin/python3 + // Possible both of the above are symlinks and point to the same file. + // Hence sort on length of the path. + // So that we process generic python3 before python3.10 + python_executables.sort_by(|a, b| { + a.to_str() + .unwrap_or_default() + .len() + .cmp(&b.to_str().unwrap_or_default().len()) + }); + let mut already_found: HashMap> = HashMap::new(); + python_executables.into_iter().for_each(|full_path| { + let version = utils::get_version_using_pyvenv_cfg(&full_path); + let possible_symlink = match utils::get_version_using_pyvenv_cfg(&full_path) { + Some(_) => { + // We got a version from pyvenv.cfg file, that means we're looking at a virtual env. + // This should not happen. + warn!( + "Found a virtual env but identified as global Python: {:?}", + full_path + ); + // Its already fully resolved as we managed to get the env version from a pyvenv.cfg in current dir. + full_path.clone() + } + None => { + is_symlinked_python_executable(&full_path.clone()).unwrap_or(full_path.clone()) } - }); + }; + let is_a_symlink = &possible_symlink != &full_path; + if already_found.contains_key(&possible_symlink) { + // If we have a symlinked file then, ensure the original path is added as symlink. + // Possible we only added /usr/local/bin/python3.10 and not /usr/local/bin/python3 + // This entry is /usr/local/bin/python3 + if is_a_symlink { + if let Some(existing) = already_found.get_mut(&full_path) { + let mut existing = existing.borrow_mut(); + if let Some(ref mut symlinks) = existing.symlinks { + symlinks.push(full_path.clone()); + } else { + existing.symlinks = + Some(vec![possible_symlink.clone(), full_path.clone()]); + } + + existing.python_executable_path = get_shortest_python_executable( + &existing.symlinks, + &existing.python_executable_path, + ); + } + } + return; + } + + if let Some(env) = self.resolve(&PythonEnv::new(full_path.clone(), None, version)) { + let mut env = env.clone(); + let mut symlinks: Option> = None; + if is_a_symlink { + symlinks = Some(vec![possible_symlink.clone(), full_path.clone()]); + } + env.python_executable_path = + get_shortest_python_executable(&symlinks, &env.python_executable_path); + env.symlinks = symlinks.clone(); + let env = RefCell::new(env); + already_found.insert(full_path, env.clone()); + if let Some(symlinks) = symlinks.clone() { + for symlink in symlinks { + already_found.insert(symlink.clone(), env.clone()); + } + } + } + }); - if environments.is_empty() { + if already_found.is_empty() { None } else { Some(LocatorResult { - environments, + environments: already_found.values().map(|v| v.borrow().clone()).collect(), managers: vec![], }) } diff --git a/native_locator/src/global_virtualenvs.rs b/native_locator/src/global_virtualenvs.rs index e0e4cf8cb991..94d4529fce35 100644 --- a/native_locator/src/global_virtualenvs.rs +++ b/native_locator/src/global_virtualenvs.rs @@ -3,7 +3,7 @@ use crate::{ known, - utils::{find_python_binary_path, get_version, PythonEnv}, + utils::{find_python_binary_path, get_version_using_pyvenv_cfg, PythonEnv}, }; use std::{fs, path::PathBuf}; @@ -57,7 +57,7 @@ pub fn list_global_virtual_envs(environment: &impl known::Environment) -> Vec Vec match captures.get(1) { Some(version) => { let version = version.as_str().to_string(); - // Never include `/opt/homebrew/bin/python` into this list. - // Yes its possible that the file `/opt/homebrew/bin/python` is a symlink to this same version. - // However what happens if user installed 3.10 and 3.11/ - // Then /opt/homebrew/bin/python will most likely point to 3.11, thats fine. - // Now assume we return the path `/opt/homebrew/bin/python` as a symlink to 3.11. - // Then user installs 3.12, how we will end up looking at the symlinks and treat - // /opt/homebrew/bin/python as 3.11, when in fact its entirely possible that - // during the installtion of 3.12, that symlink was updated to point to 3.12. - // Hence in such cases we just rely on `resolve` to always return the right information. - // & we never deal with those paths. - vec![ + let mut symlinks = vec![ PathBuf::from(format!("/opt/homebrew/bin/python{}", version)), PathBuf::from(format!("/opt/homebrew/opt/python3/bin/python{}",version)), PathBuf::from(format!("/opt/homebrew/Cellar/python@{}/{}/bin/python{}",version, full_version, version)), @@ -150,7 +140,23 @@ fn get_known_symlinks(python_exe: &PathBuf, full_version: &String) -> Vec vec![], }, @@ -163,6 +169,7 @@ fn get_known_symlinks(python_exe: &PathBuf, full_version: &String) -> Vec match captures.get(1) { @@ -180,6 +187,10 @@ fn get_known_symlinks(python_exe: &PathBuf, full_version: &String) -> Vec Vec Vec Vec { + let mut paths = env::split_paths(&self.get_env_var("PATH".to_string()).unwrap_or_default()) + .collect::>(); + vec![ - PathBuf::from("/usr/bin"), - PathBuf::from("/usr/local/bin"), PathBuf::from("/bin"), - PathBuf::from("/home/bin"), + PathBuf::from("/etc"), + PathBuf::from("/lib"), + PathBuf::from("/lib/x86_64-linux-gnu"), + PathBuf::from("/lib64"), PathBuf::from("/sbin"), - PathBuf::from("/usr/sbin"), + PathBuf::from("/snap/bin"), + PathBuf::from("/usr/bin"), + PathBuf::from("/usr/games"), + PathBuf::from("/usr/include"), + PathBuf::from("/usr/lib"), + PathBuf::from("/usr/lib/x86_64-linux-gnu"), + PathBuf::from("/usr/lib64"), + PathBuf::from("/usr/libexec"), + PathBuf::from("/usr/local"), + PathBuf::from("/usr/local/bin"), + PathBuf::from("/usr/local/etc"), + PathBuf::from("/usr/local/games"), + PathBuf::from("/usr/local/lib"), PathBuf::from("/usr/local/sbin"), + PathBuf::from("/usr/sbin"), + PathBuf::from("/usr/share"), + PathBuf::from("~/.local/bin"), + PathBuf::from("/home/bin"), PathBuf::from("/home/sbin"), PathBuf::from("/opt"), PathBuf::from("/opt/bin"), PathBuf::from("/opt/sbin"), - PathBuf::from("/opt/homebrew/bin"), ] + .iter() + .for_each(|p| { + if !paths.contains(p) { + paths.push(p.clone()); + } + }); + + paths } } diff --git a/native_locator/src/messaging.rs b/native_locator/src/messaging.rs index d66281067962..bdd323fec972 100644 --- a/native_locator/src/messaging.rs +++ b/native_locator/src/messaging.rs @@ -263,6 +263,13 @@ impl MessageDispatcher for JsonRpcDispatcher { } if !self.reported_environments.contains(&key) { self.reported_environments.insert(key); + if let Some(ref symlinks) = env.symlinks { + for symlink in symlinks { + if let Some(key) = symlink.as_os_str().to_str() { + self.reported_environments.insert(key.to_string()); + } + } + } // Get the creation and modified times. let mut env = env.clone(); diff --git a/native_locator/src/pyenv.rs b/native_locator/src/pyenv.rs index de7a9c1e69a5..a83a704ce709 100644 --- a/native_locator/src/pyenv.rs +++ b/native_locator/src/pyenv.rs @@ -41,16 +41,6 @@ fn get_home_pyenv_dir(environment: &dyn known::Environment) -> Option { } fn get_binary_from_known_paths(environment: &dyn known::Environment) -> Option { - for known_path in env::split_paths( - &environment - .get_env_var("PATH".to_string()) - .unwrap_or_default(), - ) { - let bin = PathBuf::from(known_path).join("pyenv"); - if bin.exists() { - return Some(bin); - } - } for known_path in environment.get_know_global_search_locations() { let bin = known_path.join("pyenv"); if bin.exists() { diff --git a/native_locator/src/utils.rs b/native_locator/src/utils.rs index 66bcc39c6135..2200aec7f433 100644 --- a/native_locator/src/utils.rs +++ b/native_locator/src/utils.rs @@ -89,7 +89,7 @@ pub fn find_and_parse_pyvenv_cfg(python_executable: &PathBuf) -> Option Option { +pub fn get_version_using_pyvenv_cfg(python_executable: &PathBuf) -> Option { if let Some(parent_folder) = python_executable.parent() { if let Some(pyenv_cfg) = find_and_parse_pyvenv_cfg(&parent_folder.to_path_buf()) { return Some(pyenv_cfg.version); @@ -128,25 +128,44 @@ pub fn find_python_binary_path(env_path: &Path) -> Option { None } -pub fn list_python_environments(path: &PathBuf) -> Option> { - let mut python_envs: Vec = vec![]; - for venv_dir in fs::read_dir(path).ok()? { - if let Ok(venv_dir) = venv_dir { - let venv_dir = venv_dir.path(); - if !venv_dir.is_dir() { - continue; - } - if let Some(executable) = find_python_binary_path(&venv_dir) { - python_envs.push(PythonEnv::new( - executable.clone(), - Some(venv_dir), - get_version(&executable), - )); +fn is_python_exe_name(exe: &PathBuf) -> bool { + let name = exe + .file_name() + .unwrap_or_default() + .to_str() + .unwrap_or_default() + .to_lowercase(); + if !name.starts_with("python") { + return false; + } + // Regex to match pythonX.X.exe + #[cfg(windows)] + let version_regex = Regex::new(r"python(\d+\.?)*.exe").unwrap(); + #[cfg(unix)] + let version_regex = Regex::new(r"python(\d+\.?)*$").unwrap(); + version_regex.is_match(&name) +} + +pub fn find_all_python_binaries_in_path(env_path: &Path) -> Vec { + let mut python_executables = vec![]; + #[cfg(windows)] + let bin = "Scripts"; + #[cfg(unix)] + let bin = "bin"; + let mut env_path = env_path.to_path_buf(); + if env_path.join(bin).metadata().is_ok() { + env_path = env_path.join(bin); + } + // Enumerate this directory and get all `python` & `pythonX.X` files. + if let Ok(entries) = fs::read_dir(env_path) { + for entry in entries.filter_map(Result::ok) { + let file = entry.path(); + if is_python_exe_name(&file) && file.is_file() { + python_executables.push(file); } } } - - Some(python_envs) + python_executables } pub fn get_environment_key(env: &PythonEnvironment) -> Option { @@ -194,3 +213,24 @@ pub fn get_version_from_header_files(path: &Path) -> Option { } None } + +pub fn get_shortest_python_executable( + symlinks: &Option>, + exe: &Option, +) -> Option { + // Ensure the executable always points to the shorted path. + if let Some(mut symlinks) = symlinks.clone() { + if let Some(exe) = exe { + symlinks.push(exe.clone()); + } + symlinks.sort_by(|a, b| { + a.to_str() + .unwrap_or_default() + .len() + .cmp(&b.to_str().unwrap_or_default().len()) + }); + Some(symlinks[0].clone()) + } else { + exe.clone() + } +} diff --git a/native_locator/src/virtualenvwrapper.rs b/native_locator/src/virtualenvwrapper.rs index 9a06fc2494cb..a4a30c3fe755 100644 --- a/native_locator/src/virtualenvwrapper.rs +++ b/native_locator/src/virtualenvwrapper.rs @@ -3,7 +3,7 @@ use crate::locator::{Locator, LocatorResult}; use crate::messaging::PythonEnvironment; -use crate::utils::list_python_environments; +use crate::utils::{find_python_binary_path, get_version_using_pyvenv_cfg}; use crate::virtualenv; use crate::{known::Environment, utils::PythonEnv}; use std::fs; @@ -79,6 +79,27 @@ fn get_project(env: &PythonEnv) -> Option { None } +fn list_python_environments(path: &PathBuf) -> Option> { + let mut python_envs: Vec = vec![]; + for venv_dir in fs::read_dir(path).ok()? { + if let Ok(venv_dir) = venv_dir { + let venv_dir = venv_dir.path(); + if !venv_dir.is_dir() { + continue; + } + if let Some(executable) = find_python_binary_path(&venv_dir) { + python_envs.push(PythonEnv::new( + executable.clone(), + Some(venv_dir), + get_version_using_pyvenv_cfg(&executable), + )); + } + } + } + + Some(python_envs) +} + pub struct VirtualEnvWrapper<'a> { pub environment: &'a dyn Environment, }