Skip to content

Commit

Permalink
Query interpreter to determine correct virtualenv paths (#2188)
Browse files Browse the repository at this point in the history 
## Summary

This PR migrates our virtualenv creation from a setup that assumes prior
knowledge of the correct paths, to a technique borrowed from
`virtualenv` whereby we use `sysconfig` and `distutils` to determine the
paths. The general trick is to grab the expected paths with `sysconfig`,
then make them all relative, then make them absolute for a given
directory.

Closes #2095.
Closes #2153.
  • Loading branch information
@charliermarsh
charliermarsh committed Mar 5, 2024
1 parent a8ac7b1 commit 0f6fc11
Show file tree
Hide file tree
Showing 6 changed files with 171 additions and 66 deletions.
1 change: 1 addition & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion crates/pypi-types/src/scheme.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ use serde::{Deserialize, Serialize};
/// See: <https://docs.python.org/3.12/library/sysconfig.html#installation-paths>
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct Scheme {
pub stdlib: PathBuf,
pub purelib: PathBuf,
pub platlib: PathBuf,
pub scripts: PathBuf,
Expand Down
124 changes: 121 additions & 3 deletions crates/uv-interpreter/src/get_interpreter_info.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
3: Python version 3 or newer is required
"""


import json
import os
import platform
Expand Down Expand Up @@ -67,9 +66,126 @@ def format_full_version(info):
# [3]: https://github.com/pypa/packaging/issues/678#issuecomment-1436033646
# [4]: https://github.com/astral-sh/uv/issues/1357#issuecomment-1947645243
# [5]: https://github.com/pypa/packaging/blob/085ff41692b687ae5b0772a55615b69a5b677be9/packaging/version.py#L168-L193
if len(python_full_version) > 0 and python_full_version[-1] == '+':
if len(python_full_version) > 0 and python_full_version[-1] == "+":
python_full_version = python_full_version[:-1]


def get_virtualenv():
"""Return the expected Scheme for virtualenvs created by this interpreter.
The paths returned should be relative to a root directory.
This is based on virtualenv's path discovery logic:
https://github.com/pypa/virtualenv/blob/5cd543fdf8047600ff2737babec4a635ad74d169/src/virtualenv/discovery/py_info.py#L80C9-L80C17
"""
scheme_names = sysconfig.get_scheme_names()

# Determine the scheme to use, if any.
if "venv" in scheme_names:
sysconfig_scheme = "venv"
elif sys.version_info[:2] == (3, 10) and "deb_system" in scheme_names:
# debian / ubuntu python 3.10 without `python3-distutils` will report
# mangled `local/bin` / etc. names for the default prefix
# intentionally select `posix_prefix` which is the unaltered posix-like paths
sysconfig_scheme = "posix_prefix"
else:
sysconfig_scheme = None

# Use `sysconfig`, if available.
if sysconfig_scheme:
import re

sysconfig_paths = {
i: sysconfig.get_path(i, expand=False, scheme=sysconfig_scheme)
for i in sysconfig.get_path_names()
}

# Determine very configuration variable that we need to resolve.
config_var_keys = set()

conf_var_re = re.compile(r"\{\w+}")
for element in sysconfig_paths.values():
for k in conf_var_re.findall(element):
config_var_keys.add(k[1:-1])
config_var_keys.add("PYTHONFRAMEWORK")

# Look them up.
sysconfig_vars = {i: sysconfig.get_config_var(i or "") for i in config_var_keys}

# Information about the prefix (determines the Python home).
prefix = os.path.abspath(sys.prefix)
base_prefix = os.path.abspath(sys.base_prefix)

# Information about the exec prefix (dynamic stdlib modules).
base_exec_prefix = os.path.abspath(sys.base_exec_prefix)
exec_prefix = os.path.abspath(sys.exec_prefix)

# Set any prefixes to empty, which makes the resulting paths relative.
prefixes = prefix, exec_prefix, base_prefix, base_exec_prefix
sysconfig_vars.update(
{k: "" if v in prefixes else v for k, v in sysconfig_vars.items()}
)

def expand_path(path: str) -> str:
return path.format(**sysconfig_vars).replace("/", os.sep).lstrip(os.sep)

return {
"purelib": expand_path(sysconfig_paths["purelib"]),
"platlib": expand_path(sysconfig_paths["platlib"]),
"include": expand_path(sysconfig_paths["include"]),
"scripts": expand_path(sysconfig_paths["scripts"]),
"data": expand_path(sysconfig_paths["data"]),
}
else:
# Use distutils primarily because that's what pip does.
# https://github.com/pypa/pip/blob/ae5fff36b0aad6e5e0037884927eaa29163c0611/src/pip/_internal/locations/__init__.py#L249
import warnings

with warnings.catch_warnings(): # disable warning for PEP-632
warnings.simplefilter("ignore")
from distutils import dist
from distutils.command.install import SCHEME_KEYS

d = dist.Distribution({"script_args": "--no-user-cfg"})
if hasattr(sys, "_framework"):
sys._framework = None

with warnings.catch_warnings():
warnings.simplefilter("ignore")
i = d.get_command_obj("install", create=True)

i.prefix = os.sep
i.finalize_options()
distutils_paths = {
key: (getattr(i, f"install_{key}")[1:]).lstrip(os.sep)
for key in SCHEME_KEYS
}

return {
"purelib": distutils_paths["purelib"],
"platlib": distutils_paths["platlib"],
"include": os.path.dirname(distutils_paths["headers"]),
"scripts": distutils_paths["scripts"],
"data": distutils_paths["data"],
}


def get_scheme():
"""Return the Scheme for the current interpreter.
The paths returned should be absolute.
"""
# TODO(charlie): Use distutils on required Python distributions.
paths = sysconfig.get_paths()
return {
"purelib": paths["purelib"],
"platlib": paths["platlib"],
"include": paths["include"],
"scripts": paths["scripts"],
"data": paths["data"],
}


markers = {
"implementation_name": implementation_name,
"implementation_version": implementation_version,
Expand All @@ -90,6 +206,8 @@ def format_full_version(info):
"prefix": sys.prefix,
"base_executable": getattr(sys, "_base_executable", None),
"sys_executable": sys.executable,
"scheme": sysconfig.get_paths(),
"stdlib": sysconfig.get_path("stdlib"),
"scheme": get_scheme(),
"virtualenv": get_virtualenv(),
}
print(json.dumps(interpreter_info))
55 changes: 33 additions & 22 deletions crates/uv-interpreter/src/interpreter.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,13 @@ pub struct Interpreter {
platform: Platform,
markers: Box<MarkerEnvironment>,
scheme: Scheme,
virtualenv: Scheme,
prefix: PathBuf,
base_exec_prefix: PathBuf,
base_prefix: PathBuf,
base_executable: Option<PathBuf>,
sys_executable: PathBuf,
stdlib: PathBuf,
tags: OnceCell<Tags>,
}

Expand All @@ -51,11 +53,13 @@ impl Interpreter {
platform,
markers: Box::new(info.markers),
scheme: info.scheme,
virtualenv: info.virtualenv,
prefix: info.prefix,
base_exec_prefix: info.base_exec_prefix,
base_prefix: info.base_prefix,
base_executable: info.base_executable,
sys_executable: info.sys_executable,
stdlib: info.stdlib,
tags: OnceCell::new(),
})
}
Expand All @@ -66,7 +70,13 @@ impl Interpreter {
platform,
markers: Box::new(markers),
scheme: Scheme {
stdlib: PathBuf::from("/dev/null"),
purelib: PathBuf::from("/dev/null"),
platlib: PathBuf::from("/dev/null"),
include: PathBuf::from("/dev/null"),
scripts: PathBuf::from("/dev/null"),
data: PathBuf::from("/dev/null"),
},
virtualenv: Scheme {
purelib: PathBuf::from("/dev/null"),
platlib: PathBuf::from("/dev/null"),
include: PathBuf::from("/dev/null"),
Expand All @@ -78,6 +88,7 @@ impl Interpreter {
base_prefix: PathBuf::from("/dev/null"),
base_executable: None,
sys_executable: PathBuf::from("/dev/null"),
stdlib: PathBuf::from("/dev/null"),
tags: OnceCell::new(),
}
}
Expand Down Expand Up @@ -240,7 +251,7 @@ impl Interpreter {
return None;
}

let Ok(contents) = fs::read_to_string(self.scheme.stdlib.join("EXTERNALLY-MANAGED")) else {
let Ok(contents) = fs::read_to_string(self.stdlib.join("EXTERNALLY-MANAGED")) else {
return None;
};

Expand Down Expand Up @@ -345,6 +356,11 @@ impl Interpreter {
&self.sys_executable
}

/// Return the `stdlib` path for this Python interpreter, as returned by `sysconfig.get_paths()`.
pub fn stdlib(&self) -> &Path {
&self.stdlib
}

/// Return the `purelib` path for this Python interpreter, as returned by `sysconfig.get_paths()`.
pub fn purelib(&self) -> &Path {
&self.scheme.purelib
Expand All @@ -370,21 +386,9 @@ impl Interpreter {
&self.scheme.include
}

/// Return the `stdlib` path for this Python interpreter, as returned by `sysconfig.get_paths()`.
pub fn stdlib(&self) -> &Path {
&self.scheme.stdlib
}

/// Return the name of the Python directory used to build the path to the
/// `site-packages` directory.
///
/// If one could not be determined, then `python` is returned.
pub fn site_packages_python(&self) -> &str {
if self.implementation_name() == "pypy" {
"pypy"
} else {
"python"
}
/// Return the [`Scheme`] for a virtual environment created by this [`Interpreter`].
pub fn virtualenv(&self) -> &Scheme {
&self.virtualenv
}

/// Return the [`Layout`] environment used to install wheels into this interpreter.
Expand All @@ -394,7 +398,6 @@ impl Interpreter {
sys_executable: self.sys_executable().to_path_buf(),
os_name: self.markers.os_name.clone(),
scheme: Scheme {
stdlib: self.stdlib().to_path_buf(),
purelib: self.purelib().to_path_buf(),
platlib: self.platlib().to_path_buf(),
scripts: self.scripts().to_path_buf(),
Expand All @@ -403,8 +406,7 @@ impl Interpreter {
// If the interpreter is a venv, then the `include` directory has a different structure.
// See: https://github.com/pypa/pip/blob/0ad4c94be74cc24874c6feb5bb3c2152c398a18e/src/pip/_internal/locations/_sysconfig.py#L172
self.prefix.join("include").join("site").join(format!(
"{}{}.{}",
self.site_packages_python(),
"python{}.{}",
self.python_major(),
self.python_minor()
))
Expand Down Expand Up @@ -435,11 +437,13 @@ impl ExternallyManaged {
struct InterpreterInfo {
markers: MarkerEnvironment,
scheme: Scheme,
virtualenv: Scheme,
prefix: PathBuf,
base_exec_prefix: PathBuf,
base_prefix: PathBuf,
base_executable: Option<PathBuf>,
sys_executable: PathBuf,
stdlib: PathBuf,
}

impl InterpreterInfo {
Expand Down Expand Up @@ -635,13 +639,20 @@ mod tests {
"base_prefix": "/home/ferris/.pyenv/versions/3.12.0",
"prefix": "/home/ferris/projects/uv/.venv",
"sys_executable": "/home/ferris/projects/uv/.venv/bin/python",
"stdlib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12",
"scheme": {
"data": "/home/ferris/.pyenv/versions/3.12.0",
"include": "/home/ferris/.pyenv/versions/3.12.0/include",
"platlib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12/site-packages",
"purelib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12/site-packages",
"scripts": "/home/ferris/.pyenv/versions/3.12.0/bin",
"stdlib": "/home/ferris/.pyenv/versions/3.12.0/lib/python3.12"
"scripts": "/home/ferris/.pyenv/versions/3.12.0/bin"
},
"virtualenv": {
"data": "",
"include": "include",
"platlib": "lib/python3.12/site-packages",
"purelib": "lib/python3.12/site-packages",
"scripts": "bin"
}
}
"##};
Expand Down
1 change: 1 addition & 0 deletions crates/uv-virtualenv/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ cachedir = { workspace = true }
clap = { workspace = true, features = ["derive"], optional = true }
directories = { workspace = true }
fs-err = { workspace = true }
pathdiff = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
tempfile = { workspace = true }
Expand Down
Loading

0 comments on commit 0f6fc11

Please sign in to comment.