Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add --target support to sync and install #3257

Merged
merged 4 commits into from
Apr 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions crates/uv-interpreter/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ uv-warnings = { workspace = true }

configparser = { workspace = true }
fs-err = { workspace = true, features = ["tokio"] }
itertools = { workspace = true }
once_cell = { workspace = true }
regex = { workspace = true }
rmp-serde = { workspace = true }
Expand Down
68 changes: 51 additions & 17 deletions crates/uv-interpreter/src/interpreter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@ use uv_cache::{Cache, CacheBucket, CachedByTimestamp, Freshness, Timestamp};
use uv_fs::{write_atomic_sync, PythonExt, Simplified};
use uv_toolchain::PythonVersion;

use crate::Error;
use crate::Virtualenv;
use crate::{Error, Target};

/// A Python executable and its associated platform markers.
#[derive(Debug, Clone)]
Expand All @@ -35,6 +35,7 @@ pub struct Interpreter {
sys_executable: PathBuf,
stdlib: PathBuf,
tags: OnceCell<Tags>,
target: Option<Target>,
gil_disabled: bool,
}

Expand Down Expand Up @@ -62,6 +63,7 @@ impl Interpreter {
sys_executable: info.sys_executable,
stdlib: info.stdlib,
tags: OnceCell::new(),
target: None,
})
}

Expand Down Expand Up @@ -91,6 +93,7 @@ impl Interpreter {
sys_executable: PathBuf::from("/dev/null"),
stdlib: PathBuf::from("/dev/null"),
tags: OnceCell::new(),
target: None,
gil_disabled: false,
}
}
Expand All @@ -106,6 +109,17 @@ impl Interpreter {
}
}

/// Return a new [`Interpreter`] to install into the given `--target` directory.
///
/// Initializes the `--target` directory with the expected layout.
#[must_use]
pub fn with_target(self, target: Target) -> Self {
Self {
target: Some(target),
..self
}
}

/// Returns the path to the Python virtual environment.
#[inline]
pub fn platform(&self) -> &Platform {
Expand Down Expand Up @@ -135,9 +149,15 @@ impl Interpreter {
///
/// See: <https://github.com/pypa/pip/blob/0ad4c94be74cc24874c6feb5bb3c2152c398a18e/src/pip/_internal/utils/virtualenv.py#L14>
pub fn is_virtualenv(&self) -> bool {
// Maybe this should return `false` if it's a target?
self.prefix != self.base_prefix
}

/// Returns `true` if the environment is a `--target` environment.
pub fn is_target(&self) -> bool {
self.target.is_some()
}

/// Returns `Some` if the environment is externally managed, optionally including an error
/// message from the `EXTERNALLY-MANAGED` file.
///
Expand All @@ -148,6 +168,11 @@ impl Interpreter {
return None;
}

// If we're installing into a target directory, it's never externally managed.
if self.is_target() {
return None;
}

let Ok(contents) = fs::read_to_string(self.stdlib.join("EXTERNALLY-MANAGED")) else {
return None;
};
Expand Down Expand Up @@ -303,28 +328,37 @@ impl Interpreter {
self.gil_disabled
}

/// Return the `--target` directory for this interpreter, if any.
pub fn target(&self) -> Option<&Target> {
self.target.as_ref()
}

/// Return the [`Layout`] environment used to install wheels into this interpreter.
pub fn layout(&self) -> Layout {
Layout {
python_version: self.python_tuple(),
sys_executable: self.sys_executable().to_path_buf(),
os_name: self.markers.os_name.clone(),
scheme: Scheme {
purelib: self.purelib().to_path_buf(),
platlib: self.platlib().to_path_buf(),
scripts: self.scripts().to_path_buf(),
data: self.data().to_path_buf(),
include: if self.is_virtualenv() {
// If the interpreter is a venv, then the `include` directory has a different structure.
// See: https://github.com/pypa/pip/blob/0ad4c94be74cc24874c6feb5bb3c2152c398a18e/src/pip/_internal/locations/_sysconfig.py#L172
self.prefix.join("include").join("site").join(format!(
"python{}.{}",
self.python_major(),
self.python_minor()
))
} else {
self.include().to_path_buf()
},
scheme: if let Some(target) = self.target.as_ref() {
target.scheme()
} else {
Scheme {
purelib: self.purelib().to_path_buf(),
platlib: self.platlib().to_path_buf(),
scripts: self.scripts().to_path_buf(),
data: self.data().to_path_buf(),
include: if self.is_virtualenv() {
// If the interpreter is a venv, then the `include` directory has a different structure.
// See: https://github.com/pypa/pip/blob/0ad4c94be74cc24874c6feb5bb3c2152c398a18e/src/pip/_internal/locations/_sysconfig.py#L172
self.prefix.join("include").join("site").join(format!(
"python{}.{}",
self.python_major(),
self.python_minor()
))
} else {
self.include().to_path_buf()
},
}
},
}
}
Expand Down
2 changes: 2 additions & 0 deletions crates/uv-interpreter/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,14 @@ pub use crate::find_python::{find_best_python, find_default_python, find_request
pub use crate::interpreter::Interpreter;
use crate::interpreter::InterpreterInfoError;
pub use crate::python_environment::PythonEnvironment;
pub use crate::target::Target;
pub use crate::virtualenv::Virtualenv;

mod cfg;
mod find_python;
mod interpreter;
mod python_environment;
mod target;
mod virtualenv;

#[derive(Debug, Error)]
Expand Down
44 changes: 32 additions & 12 deletions crates/uv-interpreter/src/python_environment.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
use itertools::Either;
use std::env;
use std::path::{Path, PathBuf};

Expand All @@ -8,7 +9,7 @@ use uv_cache::Cache;
use uv_fs::{LockedFile, Simplified};

use crate::cfg::PyVenvConfiguration;
use crate::{find_default_python, find_requested_python, Error, Interpreter};
use crate::{find_default_python, find_requested_python, Error, Interpreter, Target};

/// A Python environment, consisting of a Python [`Interpreter`] and its associated paths.
#[derive(Debug, Clone)]
Expand Down Expand Up @@ -68,7 +69,16 @@ impl PythonEnvironment {
}
}

/// Returns the location of the Python interpreter.
/// Create a [`PythonEnvironment`] from an existing [`Interpreter`] and `--target` directory.
#[must_use]
pub fn with_target(self, target: Target) -> Self {
Self {
interpreter: self.interpreter.with_target(target),
..self
}
}

/// Returns the root (i.e., `prefix`) of the Python interpreter.
pub fn root(&self) -> &Path {
&self.root
}
Expand Down Expand Up @@ -97,15 +107,19 @@ impl PythonEnvironment {
/// Some distributions also create symbolic links from `purelib` to `platlib`; in such cases, we
/// still deduplicate the entries, returning a single path.
pub fn site_packages(&self) -> impl Iterator<Item = &Path> {
let purelib = self.interpreter.purelib();
let platlib = self.interpreter.platlib();
std::iter::once(purelib).chain(
if purelib == platlib || is_same_file(purelib, platlib).unwrap_or(false) {
None
} else {
Some(platlib)
},
)
if let Some(target) = self.interpreter.target() {
Either::Left(std::iter::once(target.root()))
} else {
let purelib = self.interpreter.purelib();
let platlib = self.interpreter.platlib();
Either::Right(std::iter::once(purelib).chain(
if purelib == platlib || is_same_file(purelib, platlib).unwrap_or(false) {
None
} else {
Some(platlib)
},
))
}
}

/// Returns the path to the `bin` directory inside a virtual environment.
Expand All @@ -115,7 +129,13 @@ impl PythonEnvironment {

/// Grab a file lock for the virtual environment to prevent concurrent writes across processes.
pub fn lock(&self) -> Result<LockedFile, std::io::Error> {
if self.interpreter.is_virtualenv() {
if let Some(target) = self.interpreter.target() {
// If we're installing into a `--target`, use a target-specific lock file.
LockedFile::acquire(
target.root().join(".lock"),
target.root().simplified_display(),
)
} else if self.interpreter.is_virtualenv() {
// If the environment a virtualenv, use a virtualenv-specific lock file.
LockedFile::acquire(self.root.join(".lock"), self.root.simplified_display())
} else {
Expand Down
40 changes: 40 additions & 0 deletions crates/uv-interpreter/src/target.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
use std::path::{Path, PathBuf};

use pypi_types::Scheme;

/// A `--target` directory into which packages can be installed, separate from a virtual environment
/// or system Python interpreter.
#[derive(Debug, Clone)]
pub struct Target(PathBuf);

impl Target {
/// Return the [`Scheme`] for the `--target` directory.
pub fn scheme(&self) -> Scheme {
Scheme {
purelib: self.0.clone(),
platlib: self.0.clone(),
scripts: self.0.join("bin"),
data: self.0.clone(),
include: self.0.join("include"),
}
}

/// Initialize the `--target` directory.
pub fn init(&self) -> std::io::Result<()> {
fs_err::create_dir_all(&self.0)?;
fs_err::create_dir_all(self.0.join("bin"))?;
fs_err::create_dir_all(self.0.join("include"))?;
Ok(())
}

/// Return the path to the `--target` directory.
pub fn root(&self) -> &Path {
&self.0
}
}

impl From<PathBuf> for Target {
fn from(path: PathBuf) -> Self {
Self(path)
}
}
1 change: 1 addition & 0 deletions crates/uv-workspace/src/settings.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ pub struct PipOptions {
pub python: Option<String>,
pub system: Option<bool>,
pub break_system_packages: Option<bool>,
pub target: Option<PathBuf>,
pub offline: Option<bool>,
pub index_url: Option<IndexUrl>,
pub extra_index_url: Option<Vec<IndexUrl>>,
Expand Down
15 changes: 15 additions & 0 deletions crates/uv/src/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -768,6 +768,11 @@ pub(crate) struct PipSyncArgs {
#[arg(long, overrides_with("break_system_packages"))]
pub(crate) no_break_system_packages: bool,

/// Install packages into the specified directory, rather than into the virtual environment
/// or system Python interpreter.
#[arg(long)]
pub(crate) target: Option<PathBuf>,

/// Use legacy `setuptools` behavior when building source distributions without a
/// `pyproject.toml`.
#[arg(long, overrides_with("no_legacy_setup_py"))]
Expand Down Expand Up @@ -1132,6 +1137,11 @@ pub(crate) struct PipInstallArgs {
#[arg(long, overrides_with("break_system_packages"))]
pub(crate) no_break_system_packages: bool,

/// Install packages into the specified directory, rather than into the virtual environment
/// or system Python interpreter.
#[arg(long)]
pub(crate) target: Option<PathBuf>,

/// Use legacy `setuptools` behavior when building source distributions without a
/// `pyproject.toml`.
#[arg(long, overrides_with("no_legacy_setup_py"))]
Expand Down Expand Up @@ -1335,6 +1345,11 @@ pub(crate) struct PipUninstallArgs {
#[arg(long, overrides_with("break_system_packages"))]
pub(crate) no_break_system_packages: bool,

/// Uninstall packages from the specified directory, rather than from the virtual environment
/// or system Python interpreter.
#[arg(long)]
pub(crate) target: Option<PathBuf>,

/// Run offline, i.e., without accessing the network.
#[arg(long, overrides_with("no_offline"))]
pub(crate) offline: bool,
Expand Down
13 changes: 10 additions & 3 deletions crates/uv/src/commands/pip_install.rs
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
use std::borrow::Cow;
use std::fmt::Write;

use std::path::Path;

use anstream::eprint;
use anyhow::{anyhow, Context, Result};

use itertools::Itertools;
use owo_colors::OwoColorize;
use tempfile::tempdir_in;
Expand Down Expand Up @@ -33,7 +31,7 @@ use uv_configuration::{KeyringProviderType, TargetTriple};
use uv_dispatch::BuildDispatch;
use uv_fs::Simplified;
use uv_installer::{BuiltEditable, Downloader, Plan, Planner, ResolvedEditable, SitePackages};
use uv_interpreter::{Interpreter, PythonEnvironment};
use uv_interpreter::{Interpreter, PythonEnvironment, Target};
use uv_normalize::PackageName;
use uv_requirements::{
ExtrasSpecification, LookaheadResolver, NamedRequirementsResolver, RequirementsSource,
Expand Down Expand Up @@ -84,6 +82,7 @@ pub(crate) async fn pip_install(
python: Option<String>,
system: bool,
break_system_packages: bool,
target: Option<Target>,
native_tls: bool,
cache: Cache,
dry_run: bool,
Expand Down Expand Up @@ -134,6 +133,14 @@ pub(crate) async fn pip_install(
venv.python_executable().user_display().cyan()
);

// Apply any `--target` directory.
let venv = if let Some(target) = target {
target.init()?;
venv.with_target(target)
} else {
venv
};

// If the environment is externally managed, abort.
if let Some(externally_managed) = venv.interpreter().is_externally_managed() {
if break_system_packages {
Expand Down
Loading
Loading