From 26263f1516bc2cc67cb4a34bbabd05385aa0bbb9 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Mon, 4 Mar 2024 17:39:58 -0500 Subject: [PATCH] Query interpreter to determine correct virtualenv paths --- Cargo.lock | 1 + crates/pypi-types/src/scheme.rs | 1 - .../src/get_interpreter_info.py | 124 +++++++++++++++++- crates/uv-interpreter/src/interpreter.rs | 55 ++++---- crates/uv-virtualenv/Cargo.toml | 1 + crates/uv-virtualenv/src/bare.rs | 55 +++----- 6 files changed, 171 insertions(+), 66 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bb6fb7596548..9ccfdfc51816 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4662,6 +4662,7 @@ dependencies = [ "clap", "directories", "fs-err", + "pathdiff", "platform-host", "pypi-types", "serde", diff --git a/crates/pypi-types/src/scheme.rs b/crates/pypi-types/src/scheme.rs index de921b3755fe..32c46518d710 100644 --- a/crates/pypi-types/src/scheme.rs +++ b/crates/pypi-types/src/scheme.rs @@ -9,7 +9,6 @@ use serde::{Deserialize, Serialize}; /// See: #[derive(Debug, Deserialize, Serialize, Clone)] pub struct Scheme { - pub stdlib: PathBuf, pub purelib: PathBuf, pub platlib: PathBuf, pub scripts: PathBuf, diff --git a/crates/uv-interpreter/src/get_interpreter_info.py b/crates/uv-interpreter/src/get_interpreter_info.py index 917f19770df6..da02e707f577 100644 --- a/crates/uv-interpreter/src/get_interpreter_info.py +++ b/crates/uv-interpreter/src/get_interpreter_info.py @@ -7,7 +7,6 @@ 3: Python version 3 or newer is required """ - import json import os import platform @@ -67,9 +66,126 @@ def format_full_version(info): # [3]: https://github.com/pypa/packaging/issues/678#issuecomment-1436033646 # [4]: https://github.com/astral-sh/uv/issues/1357#issuecomment-1947645243 # [5]: https://github.com/pypa/packaging/blob/085ff41692b687ae5b0772a55615b69a5b677be9/packaging/version.py#L168-L193 -if len(python_full_version) > 0 and python_full_version[-1] == '+': +if len(python_full_version) > 0 and python_full_version[-1] == "+": python_full_version = python_full_version[:-1] + +def get_virtualenv(): + """Return the expected Scheme for virtualenvs created by this interpreter. + + The paths returned should be relative to a root directory. + + This is based on virtualenv's path discovery logic: + https://github.com/pypa/virtualenv/blob/5cd543fdf8047600ff2737babec4a635ad74d169/src/virtualenv/discovery/py_info.py#L80C9-L80C17 + """ + scheme_names = sysconfig.get_scheme_names() + + # Determine the scheme to use, if any. + if "venv" in scheme_names: + sysconfig_scheme = "venv" + elif sys.version_info[:2] == (3, 10) and "deb_system" in scheme_names: + # debian / ubuntu python 3.10 without `python3-distutils` will report + # mangled `local/bin` / etc. names for the default prefix + # intentionally select `posix_prefix` which is the unaltered posix-like paths + sysconfig_scheme = "posix_prefix" + else: + sysconfig_scheme = None + + # Use `sysconfig`, if available. + if sysconfig_scheme: + import re + + sysconfig_paths = { + i: sysconfig.get_path(i, expand=False, scheme=sysconfig_scheme) + for i in sysconfig.get_path_names() + } + + # Determine very configuration variable that we need to resolve. + config_var_keys = set() + + conf_var_re = re.compile(r"\{\w+}") + for element in sysconfig_paths.values(): + for k in conf_var_re.findall(element): + config_var_keys.add(k[1:-1]) + config_var_keys.add("PYTHONFRAMEWORK") + + # Look them up. + sysconfig_vars = {i: sysconfig.get_config_var(i or "") for i in config_var_keys} + + # Information about the prefix (determines the Python home). + prefix = os.path.abspath(sys.prefix) + base_prefix = os.path.abspath(sys.base_prefix) + + # Information about the exec prefix (dynamic stdlib modules). + base_exec_prefix = os.path.abspath(sys.base_exec_prefix) + exec_prefix = os.path.abspath(sys.exec_prefix) + + # Set any prefixes to empty, which makes the resulting paths relative. + prefixes = prefix, exec_prefix, base_prefix, base_exec_prefix + sysconfig_vars.update( + {k: "" if v in prefixes else v for k, v in sysconfig_vars.items()} + ) + + def expand_path(path: str) -> str: + return path.format(**sysconfig_vars).replace("/", os.sep).lstrip(os.sep) + + return { + "purelib": expand_path(sysconfig_paths["purelib"]), + "platlib": expand_path(sysconfig_paths["platlib"]), + "include": expand_path(sysconfig_paths["include"]), + "scripts": expand_path(sysconfig_paths["scripts"]), + "data": expand_path(sysconfig_paths["data"]), + } + else: + # Use distutils primarily because that's what pip does. + # https://github.com/pypa/pip/blob/ae5fff36b0aad6e5e0037884927eaa29163c0611/src/pip/_internal/locations/__init__.py#L249 + import warnings + + with warnings.catch_warnings(): # disable warning for PEP-632 + warnings.simplefilter("ignore") + from distutils import dist + from distutils.command.install import SCHEME_KEYS + + d = dist.Distribution({"script_args": "--no-user-cfg"}) + if hasattr(sys, "_framework"): + sys._framework = None + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + i = d.get_command_obj("install", create=True) + + i.prefix = os.sep + i.finalize_options() + distutils_paths = { + key: (getattr(i, f"install_{key}")[1:]).lstrip(os.sep) + for key in SCHEME_KEYS + } + + return { + "purelib": distutils_paths["purelib"], + "platlib": distutils_paths["platlib"], + "include": os.path.dirname(distutils_paths["headers"]), + "scripts": distutils_paths["scripts"], + "data": distutils_paths["data"], + } + + +def get_scheme(): + """Return the Scheme for the current interpreter. + + The paths returned should be absolute. + """ + # TODO(charlie): Use distutils on required Python distributions. + paths = sysconfig.get_paths() + return { + "purelib": paths["purelib"], + "platlib": paths["platlib"], + "include": paths["include"], + "scripts": paths["scripts"], + "data": paths["data"], + } + + markers = { "implementation_name": implementation_name, "implementation_version": implementation_version, @@ -90,6 +206,8 @@ def format_full_version(info): "prefix": sys.prefix, "base_executable": getattr(sys, "_base_executable", None), "sys_executable": sys.executable, - "scheme": sysconfig.get_paths(), + "stdlib": sysconfig.get_path("stdlib"), + "scheme": get_scheme(), + "virtualenv": get_virtualenv(), } print(json.dumps(interpreter_info)) diff --git a/crates/uv-interpreter/src/interpreter.rs b/crates/uv-interpreter/src/interpreter.rs index 22c42f670ee9..5929a5caf1d3 100644 --- a/crates/uv-interpreter/src/interpreter.rs +++ b/crates/uv-interpreter/src/interpreter.rs @@ -29,11 +29,13 @@ pub struct Interpreter { platform: Platform, markers: Box, scheme: Scheme, + virtualenv: Scheme, prefix: PathBuf, base_exec_prefix: PathBuf, base_prefix: PathBuf, base_executable: Option, sys_executable: PathBuf, + stdlib: PathBuf, tags: OnceCell, } @@ -52,11 +54,13 @@ impl Interpreter { platform, markers: Box::new(info.markers), scheme: info.scheme, + virtualenv: info.virtualenv, prefix: info.prefix, base_exec_prefix: info.base_exec_prefix, base_prefix: info.base_prefix, base_executable: info.base_executable, sys_executable: info.sys_executable, + stdlib: info.stdlib, tags: OnceCell::new(), }) } @@ -67,7 +71,13 @@ impl Interpreter { platform, markers: Box::new(markers), scheme: Scheme { - stdlib: PathBuf::from("/dev/null"), + purelib: PathBuf::from("/dev/null"), + platlib: PathBuf::from("/dev/null"), + include: PathBuf::from("/dev/null"), + scripts: PathBuf::from("/dev/null"), + data: PathBuf::from("/dev/null"), + }, + virtualenv: Scheme { purelib: PathBuf::from("/dev/null"), platlib: PathBuf::from("/dev/null"), include: PathBuf::from("/dev/null"), @@ -79,6 +89,7 @@ impl Interpreter { base_prefix: PathBuf::from("/dev/null"), base_executable: None, sys_executable: PathBuf::from("/dev/null"), + stdlib: PathBuf::from("/dev/null"), tags: OnceCell::new(), } } @@ -260,7 +271,7 @@ impl Interpreter { return None; } - let Ok(contents) = fs::read_to_string(self.scheme.stdlib.join("EXTERNALLY-MANAGED")) else { + let Ok(contents) = fs::read_to_string(self.stdlib.join("EXTERNALLY-MANAGED")) else { return None; }; @@ -365,6 +376,11 @@ impl Interpreter { &self.sys_executable } + /// Return the `stdlib` path for this Python interpreter, as returned by `sysconfig.get_paths()`. + pub fn stdlib(&self) -> &Path { + &self.stdlib + } + /// Return the `purelib` path for this Python interpreter, as returned by `sysconfig.get_paths()`. pub fn purelib(&self) -> &Path { &self.scheme.purelib @@ -390,21 +406,9 @@ impl Interpreter { &self.scheme.include } - /// Return the `stdlib` path for this Python interpreter, as returned by `sysconfig.get_paths()`. - pub fn stdlib(&self) -> &Path { - &self.scheme.stdlib - } - - /// Return the name of the Python directory used to build the path to the - /// `site-packages` directory. - /// - /// If one could not be determined, then `python` is returned. - pub fn site_packages_python(&self) -> &str { - if self.implementation_name() == "pypy" { - "pypy" - } else { - "python" - } + /// Return the [`Scheme`] for a virtual environment created by this [`Interpreter`]. + pub fn virtualenv(&self) -> &Scheme { + &self.virtualenv } /// Return the [`Layout`] environment used to install wheels into this interpreter. @@ -414,7 +418,6 @@ impl Interpreter { sys_executable: self.sys_executable().to_path_buf(), os_name: self.markers.os_name.clone(), scheme: Scheme { - stdlib: self.stdlib().to_path_buf(), purelib: self.purelib().to_path_buf(), platlib: self.platlib().to_path_buf(), scripts: self.scripts().to_path_buf(), @@ -423,8 +426,7 @@ impl Interpreter { // If the interpreter is a venv, then the `include` directory has a different structure. // See: https://github.com/pypa/pip/blob/0ad4c94be74cc24874c6feb5bb3c2152c398a18e/src/pip/_internal/locations/_sysconfig.py#L172 self.prefix.join("include").join("site").join(format!( - "{}{}.{}", - self.site_packages_python(), + "python{}.{}", self.python_major(), self.python_minor() )) @@ -455,11 +457,13 @@ impl ExternallyManaged { struct InterpreterInfo { markers: MarkerEnvironment, scheme: Scheme, + virtualenv: Scheme, prefix: PathBuf, base_exec_prefix: PathBuf, base_prefix: PathBuf, base_executable: Option, sys_executable: PathBuf, + stdlib: PathBuf, } impl InterpreterInfo { @@ -655,13 +659,20 @@ mod tests { "base_prefix": "/home/ferris/.pyenv/versions/3.12.0", "prefix": "/home/ferris/projects/uv/.venv", "sys_executable": "/home/ferris/projects/uv/.venv/bin/python", + "stdlib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12", "scheme": { "data": "/home/ferris/.pyenv/versions/3.12.0", "include": "/home/ferris/.pyenv/versions/3.12.0/include", "platlib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12/site-packages", "purelib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12/site-packages", - "scripts": "/home/ferris/.pyenv/versions/3.12.0/bin", - "stdlib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12" + "scripts": "/home/ferris/.pyenv/versions/3.12.0/bin" + }, + "virtualenv": { + "data": "", + "include": "include", + "platlib": "lib/python3.12/site-packages", + "purelib": "lib/python3.12/site-packages", + "scripts": "bin" } } "##}; diff --git a/crates/uv-virtualenv/Cargo.toml b/crates/uv-virtualenv/Cargo.toml index 3e938ad83546..f4f3a660763a 100644 --- a/crates/uv-virtualenv/Cargo.toml +++ b/crates/uv-virtualenv/Cargo.toml @@ -32,6 +32,7 @@ cachedir = { workspace = true } clap = { workspace = true, features = ["derive"], optional = true } directories = { workspace = true } fs-err = { workspace = true } +pathdiff = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } tempfile = { workspace = true } diff --git a/crates/uv-virtualenv/src/bare.rs b/crates/uv-virtualenv/src/bare.rs index d4cc26ca185e..506ce4050673 100644 --- a/crates/uv-virtualenv/src/bare.rs +++ b/crates/uv-virtualenv/src/bare.rs @@ -8,9 +8,9 @@ use std::path::Path; use fs_err as fs; use fs_err::File; +use pypi_types::Scheme; use tracing::info; -use pypi_types::Scheme; use uv_fs::Simplified; use uv_interpreter::{Interpreter, Virtualenv}; @@ -124,7 +124,7 @@ pub fn create_bare_venv( } else { unimplemented!("Only Windows and Unix are supported") }; - let scripts = location.join(bin_name); + let scripts = location.join(&interpreter.virtualenv().scripts); let prompt = match prompt { Prompt::CurrentDirectoryName => env::current_dir()? .file_name() @@ -143,7 +143,6 @@ pub fn create_bare_venv( fs::create_dir(&scripts)?; let executable = scripts.join(format!("python{EXE_SUFFIX}")); - // No symlinking on Windows, at least not on a regular non-dev non-admin Windows install. #[cfg(unix)] { use fs_err::os::unix::fs::symlink; @@ -163,6 +162,7 @@ pub fn create_bare_venv( )?; } + // No symlinking on Windows, at least not on a regular non-dev non-admin Windows install. #[cfg(windows)] { // https://github.com/python/cpython/blob/d457345bbc6414db0443819290b04a9a4333313d/Lib/venv/__init__.py#L261-L267 @@ -205,18 +205,11 @@ pub fn create_bare_venv( // Add all the activate scripts for different shells for (name, template) in ACTIVATE_TEMPLATES { - let relative_site_packages = if cfg!(unix) { - format!( - "../lib/{}{}.{}/site-packages", - interpreter.site_packages_python(), - interpreter.python_major(), - interpreter.python_minor(), - ) - } else if cfg!(windows) { - "../Lib/site-packages".to_string() - } else { - unimplemented!("Only Windows and Unix are supported") - }; + let relative_site_packages = pathdiff::diff_paths( + &interpreter.virtualenv().purelib, + &interpreter.virtualenv().scripts, + ) + .expect("Failed to calculate relative path to site-packages"); let activator = template .replace( "{{ VIRTUAL_ENV_DIR }}", @@ -230,7 +223,7 @@ pub fn create_bare_venv( ) .replace( "{{ RELATIVE_SITE_PACKAGES }}", - relative_site_packages.as_str(), + relative_site_packages.simplified().to_str().unwrap(), ); fs::write(scripts.join(name), activator)?; } @@ -296,21 +289,7 @@ pub fn create_bare_venv( drop(pyvenv_cfg); // Construct the path to the `site-packages` directory. - let site_packages = if cfg!(unix) { - location - .join("lib") - .join(format!( - "{}{}.{}", - interpreter.site_packages_python(), - interpreter.python_major(), - interpreter.python_minor(), - )) - .join("site-packages") - } else if cfg!(windows) { - location.join("Lib").join("site-packages") - } else { - unimplemented!("Only Windows and Unix are supported") - }; + let site_packages = location.join(&interpreter.virtualenv().purelib); // Populate `site-packages` with a `_virtualenv.py` file. fs::create_dir_all(&site_packages)?; @@ -319,15 +298,11 @@ pub fn create_bare_venv( Ok(Virtualenv { scheme: Scheme { - // Paths that were already constructed above. - scripts, - // Set `purelib` and `platlib` to the same value. - purelib: site_packages.clone(), - platlib: site_packages, - // Inherited from the interpreter. - stdlib: interpreter.stdlib().to_path_buf(), - include: interpreter.include().to_path_buf(), - data: location.clone(), + purelib: location.join(&interpreter.virtualenv().purelib), + platlib: location.join(&interpreter.virtualenv().platlib), + scripts: location.join(&interpreter.virtualenv().scripts), + data: location.join(&interpreter.virtualenv().data), + include: location.join(&interpreter.virtualenv().include), }, root: location, executable,