From e1708689a9da3d56e730eff239ae52480a825233 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Tue, 25 Jun 2024 17:28:50 -0400 Subject: [PATCH] Add a universal resolution mode to `pip compile` (#4505) ## Summary This needs more tests and a few more changes, but checkpointing it for now. --- crates/uv-cli/src/lib.rs | 9 ++ crates/uv-resolver/src/python_requirement.rs | 34 +++++-- crates/uv-resolver/src/resolver/mod.rs | 8 +- crates/uv-settings/src/settings.rs | 1 + crates/uv/src/commands/pip/compile.rs | 37 +++++--- crates/uv/src/main.rs | 1 + crates/uv/src/settings.rs | 6 ++ crates/uv/tests/pip_compile.rs | 94 ++++++++++++++++++++ crates/uv/tests/show_settings.rs | 17 ++++ uv.schema.json | 6 ++ 10 files changed, 186 insertions(+), 27 deletions(-) diff --git a/crates/uv-cli/src/lib.rs b/crates/uv-cli/src/lib.rs index 9a02e47233f8..d5b1f81134a0 100644 --- a/crates/uv-cli/src/lib.rs +++ b/crates/uv-cli/src/lib.rs @@ -568,6 +568,15 @@ pub struct PipCompileArgs { #[arg(long)] pub python_platform: Option, + /// Perform a universal resolution, attempting to generate a single `requirements.txt` output + /// file that is compatible with all operating systems, architectures and supported Python + /// versions. + #[arg(long, overrides_with("no_universal"))] + pub universal: bool, + + #[arg(long, overrides_with("universal"), hide = true)] + pub no_universal: bool, + /// Specify a package to omit from the output resolution. Its dependencies will still be /// included in the resolution. Equivalent to pip-compile's `--unsafe-package` option. #[arg(long, alias = "unsafe-package")] diff --git a/crates/uv-resolver/src/python_requirement.rs b/crates/uv-resolver/src/python_requirement.rs index 153c4ec35f84..5035fed5fab5 100644 --- a/crates/uv-resolver/src/python_requirement.rs +++ b/crates/uv-resolver/src/python_requirement.rs @@ -1,5 +1,5 @@ -use pep440_rs::VersionSpecifiers; -use pep508_rs::StringVersion; +use pep440_rs::{Operator, Version, VersionSpecifier, VersionSpecifiers}; +use pep508_rs::{MarkerExpression, MarkerTree, MarkerValueVersion, StringVersion}; use uv_toolchain::{Interpreter, PythonVersion}; use crate::RequiresPython; @@ -59,10 +59,32 @@ impl PythonRequirement { self.target.as_ref() } - /// Return the target version of Python as a "requires python" type, - /// if available. - pub(crate) fn requires_python(&self) -> Option<&RequiresPython> { - self.target().and_then(|target| target.as_requires_python()) + /// Return a [`MarkerTree`] representing the Python requirement. + /// + /// See: [`RequiresPython::to_marker_tree`] + pub fn to_marker_tree(&self) -> MarkerTree { + let version = match &self.target { + None => self.installed.version.clone(), + Some(PythonTarget::Version(version)) => version.version.clone(), + Some(PythonTarget::RequiresPython(requires_python)) => { + return requires_python.to_marker_tree() + } + }; + + let version_major_minor_only = Version::new(version.release().iter().take(2)); + let expr_python_version = MarkerExpression::Version { + key: MarkerValueVersion::PythonVersion, + specifier: VersionSpecifier::from_version(Operator::Equal, version_major_minor_only) + .unwrap(), + }; + let expr_python_full_version = MarkerExpression::Version { + key: MarkerValueVersion::PythonFullVersion, + specifier: VersionSpecifier::from_version(Operator::Equal, version).unwrap(), + }; + MarkerTree::And(vec![ + MarkerTree::Expression(expr_python_version), + MarkerTree::Expression(expr_python_full_version), + ]) } } diff --git a/crates/uv-resolver/src/resolver/mod.rs b/crates/uv-resolver/src/resolver/mod.rs index 8bd3fb61960c..8e8a5aa3993f 100644 --- a/crates/uv-resolver/src/resolver/mod.rs +++ b/crates/uv-resolver/src/resolver/mod.rs @@ -48,7 +48,6 @@ use crate::pubgrub::{ PubGrubPriorities, PubGrubPython, PubGrubSpecifier, }; use crate::python_requirement::PythonRequirement; -use crate::requires_python::RequiresPython; use crate::resolution::ResolutionGraph; pub(crate) use crate::resolver::availability::{ IncompletePackage, ResolverVersion, UnavailablePackage, UnavailableReason, UnavailableVersion, @@ -193,12 +192,7 @@ impl let requires_python = if markers.is_some() { None } else { - Some( - python_requirement - .requires_python() - .map(RequiresPython::to_marker_tree) - .unwrap_or_else(|| MarkerTree::And(vec![])), - ) + Some(python_requirement.to_marker_tree()) }; let state = ResolverState { index: index.clone(), diff --git a/crates/uv-settings/src/settings.rs b/crates/uv-settings/src/settings.rs index 6f5282e66bb4..7ca4c2b34b8c 100644 --- a/crates/uv-settings/src/settings.rs +++ b/crates/uv-settings/src/settings.rs @@ -177,6 +177,7 @@ pub struct PipOptions { pub config_settings: Option, pub python_version: Option, pub python_platform: Option, + pub universal: Option, pub exclude_newer: Option, pub no_emit_package: Option>, pub emit_index_url: Option, diff --git a/crates/uv/src/commands/pip/compile.rs b/crates/uv/src/commands/pip/compile.rs index f9b4d8120f1b..36d62f6e0a4a 100644 --- a/crates/uv/src/commands/pip/compile.rs +++ b/crates/uv/src/commands/pip/compile.rs @@ -77,6 +77,7 @@ pub(crate) async fn pip_compile( build_options: BuildOptions, python_version: Option, python_platform: Option, + universal: bool, exclude_newer: Option, annotation_style: AnnotationStyle, link_mode: LinkMode, @@ -219,7 +220,13 @@ pub(crate) async fn pip_compile( }; // Determine the environment for the resolution. - let (tags, markers) = resolution_environment(python_version, python_platform, &interpreter)?; + let (tags, markers) = if universal { + (None, None) + } else { + let (tags, markers) = + resolution_environment(python_version, python_platform, &interpreter)?; + (Some(tags), Some(markers)) + }; // Generate, but don't enforce hashes for the requirements. let hasher = if generate_hashes { @@ -247,7 +254,7 @@ pub(crate) async fn pip_compile( .index_urls(index_locations.index_urls()) .index_strategy(index_strategy) .keyring(keyring_provider) - .markers(&markers) + .markers(interpreter.markers()) .platform(interpreter.platform()) .build(); @@ -262,7 +269,7 @@ pub(crate) async fn pip_compile( let flat_index = { let client = FlatIndexClient::new(&client, &cache); let entries = client.fetch(index_locations.flat_index()).await?; - FlatIndex::from_entries(entries, Some(&tags), &hasher, &build_options) + FlatIndex::from_entries(entries, tags.as_deref(), &hasher, &build_options) }; // Track in-flight downloads, builds, etc., across resolutions. @@ -319,8 +326,8 @@ pub(crate) async fn pip_compile( &hasher, &Reinstall::None, &upgrade, - Some(&tags), - Some(&markers), + tags.as_deref(), + markers.as_deref(), python_requirement, &client, &flat_index, @@ -368,13 +375,15 @@ pub(crate) async fn pip_compile( } if include_marker_expression { - let relevant_markers = resolution.marker_tree(&top_level_index, &markers)?; - writeln!( - writer, - "{}", - "# Pinned dependencies known to be valid for:".green() - )?; - writeln!(writer, "{}", format!("# {relevant_markers}").green())?; + if let Some(markers) = markers.as_deref() { + let relevant_markers = resolution.marker_tree(&top_level_index, markers)?; + writeln!( + writer, + "{}", + "# Pinned dependencies known to be valid for:".green() + )?; + writeln!(writer, "{}", format!("# {relevant_markers}").green())?; + } } let mut wrote_preamble = false; @@ -439,11 +448,11 @@ pub(crate) async fn pip_compile( "{}", DisplayResolutionGraph::new( &resolution, - Some(&markers), + markers.as_deref(), &no_emit_packages, generate_hashes, include_extras, - include_markers, + include_markers || universal, include_annotations, include_index_annotation, annotation_style, diff --git a/crates/uv/src/main.rs b/crates/uv/src/main.rs index 0da5fb01149e..fcfa84a7edde 100644 --- a/crates/uv/src/main.rs +++ b/crates/uv/src/main.rs @@ -282,6 +282,7 @@ async fn run() -> Result { args.settings.build_options, args.settings.python_version, args.settings.python_platform, + args.settings.universal, args.settings.exclude_newer, args.settings.annotation_style, args.settings.link_mode, diff --git a/crates/uv/src/settings.rs b/crates/uv/src/settings.rs index 63cfb0c5ec70..fffae7af8a59 100644 --- a/crates/uv/src/settings.rs +++ b/crates/uv/src/settings.rs @@ -525,6 +525,8 @@ impl PipCompileSettings { only_binary, python_version, python_platform, + universal, + no_universal, no_emit_package, emit_index_url, no_emit_index_url, @@ -583,6 +585,7 @@ impl PipCompileSettings { legacy_setup_py: flag(legacy_setup_py, no_legacy_setup_py), python_version, python_platform, + universal: flag(universal, no_universal), no_emit_package, emit_index_url: flag(emit_index_url, no_emit_index_url), emit_find_links: flag(emit_find_links, no_emit_find_links), @@ -1499,6 +1502,7 @@ pub(crate) struct PipSettings { pub(crate) config_setting: ConfigSettings, pub(crate) python_version: Option, pub(crate) python_platform: Option, + pub(crate) universal: bool, pub(crate) exclude_newer: Option, pub(crate) no_emit_package: Vec, pub(crate) emit_index_url: bool, @@ -1555,6 +1559,7 @@ impl PipSettings { config_settings, python_version, python_platform, + universal, exclude_newer, no_emit_package, emit_index_url, @@ -1686,6 +1691,7 @@ impl PipSettings { .unwrap_or_default(), python_version: args.python_version.combine(python_version), python_platform: args.python_platform.combine(python_platform), + universal: args.universal.combine(universal).unwrap_or_default(), exclude_newer: args.exclude_newer.combine(exclude_newer), no_emit_package: args .no_emit_package diff --git a/crates/uv/tests/pip_compile.rs b/crates/uv/tests/pip_compile.rs index 1e4226fd2d26..3a7d5d6147de 100644 --- a/crates/uv/tests/pip_compile.rs +++ b/crates/uv/tests/pip_compile.rs @@ -6324,6 +6324,100 @@ fn no_strip_markers_transitive_marker() -> Result<()> { Ok(()) } +/// Perform a universal resolution with a package that has a marker. +#[test] +fn universal() -> Result<()> { + let context = TestContext::new("3.12"); + let requirements_in = context.temp_dir.child("requirements.in"); + requirements_in.write_str(indoc::indoc! {r" + trio ; python_version > '3.11' + trio ; sys_platform == 'win32' + "})?; + + uv_snapshot!(context.compile() + .arg("requirements.in") + .arg("--universal"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by uv via the following command: + # uv pip compile --cache-dir [CACHE_DIR] --exclude-newer 2024-03-25T00:00:00Z requirements.in --universal + attrs==23.2.0 ; python_version > '3.11' or sys_platform == 'win32' + # via + # outcome + # trio + cffi==1.16.0 ; implementation_name != 'pypy' and os_name == 'nt' and (python_version > '3.11' or sys_platform == 'win32') + # via trio + idna==3.6 ; python_version > '3.11' or sys_platform == 'win32' + # via trio + outcome==1.3.0.post0 ; python_version > '3.11' or sys_platform == 'win32' + # via trio + pycparser==2.21 ; implementation_name != 'pypy' and os_name == 'nt' and (python_version > '3.11' or sys_platform == 'win32') + # via cffi + sniffio==1.3.1 ; python_version > '3.11' or sys_platform == 'win32' + # via trio + sortedcontainers==2.4.0 ; python_version > '3.11' or sys_platform == 'win32' + # via trio + trio==0.25.0 ; python_version > '3.11' or sys_platform == 'win32' + # via -r requirements.in + + ----- stderr ----- + Resolved 8 packages in [TIME] + "### + ); + + Ok(()) +} + +/// Perform a universal resolution with conflicting versions and markers. +#[test] +fn universal_conflicting() -> Result<()> { + let context = TestContext::new("3.12"); + let requirements_in = context.temp_dir.child("requirements.in"); + requirements_in.write_str(indoc::indoc! {r" + trio==0.25.0 ; sys_platform == 'darwin' + trio==0.10.0 ; sys_platform == 'win32' + "})?; + + uv_snapshot!(context.compile() + .arg("requirements.in") + .arg("--universal"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by uv via the following command: + # uv pip compile --cache-dir [CACHE_DIR] --exclude-newer 2024-03-25T00:00:00Z requirements.in --universal + async-generator==1.10 ; sys_platform == 'win32' + # via trio + attrs==23.2.0 ; sys_platform == 'darwin' or sys_platform == 'win32' + # via + # outcome + # trio + cffi==1.16.0 ; (implementation_name != 'pypy' and os_name == 'nt' and sys_platform == 'darwin') or (os_name == 'nt' and sys_platform == 'win32') + # via trio + idna==3.6 ; sys_platform == 'darwin' or sys_platform == 'win32' + # via trio + outcome==1.3.0.post0 ; sys_platform == 'darwin' or sys_platform == 'win32' + # via trio + pycparser==2.21 ; (implementation_name != 'pypy' and os_name == 'nt' and sys_platform == 'darwin') or (os_name == 'nt' and sys_platform == 'win32') + # via cffi + sniffio==1.3.1 ; sys_platform == 'darwin' or sys_platform == 'win32' + # via trio + sortedcontainers==2.4.0 ; sys_platform == 'darwin' or sys_platform == 'win32' + # via trio + trio==0.10.0 ; sys_platform == 'win32' + # via -r requirements.in + trio==0.25.0 ; sys_platform == 'darwin' + # via -r requirements.in + + ----- stderr ----- + Resolved 10 packages in [TIME] + "### + ); + + Ok(()) +} + /// Resolve a package from a `requirements.in` file, with a `constraints.txt` file pinning one of /// its transitive dependencies to a specific version. #[test] diff --git a/crates/uv/tests/show_settings.rs b/crates/uv/tests/show_settings.rs index fe956f6e477f..15e889d80331 100644 --- a/crates/uv/tests/show_settings.rs +++ b/crates/uv/tests/show_settings.rs @@ -140,6 +140,7 @@ fn resolve_uv_toml() -> anyhow::Result<()> { ), python_version: None, python_platform: None, + universal: false, exclude_newer: Some( ExcludeNewer( 2024-03-25T00:00:00Z, @@ -269,6 +270,7 @@ fn resolve_uv_toml() -> anyhow::Result<()> { ), python_version: None, python_platform: None, + universal: false, exclude_newer: Some( ExcludeNewer( 2024-03-25T00:00:00Z, @@ -399,6 +401,7 @@ fn resolve_uv_toml() -> anyhow::Result<()> { ), python_version: None, python_platform: None, + universal: false, exclude_newer: Some( ExcludeNewer( 2024-03-25T00:00:00Z, @@ -561,6 +564,7 @@ fn resolve_pyproject_toml() -> anyhow::Result<()> { ), python_version: None, python_platform: None, + universal: false, exclude_newer: Some( ExcludeNewer( 2024-03-25T00:00:00Z, @@ -669,6 +673,7 @@ fn resolve_pyproject_toml() -> anyhow::Result<()> { ), python_version: None, python_platform: None, + universal: false, exclude_newer: Some( ExcludeNewer( 2024-03-25T00:00:00Z, @@ -809,6 +814,7 @@ fn resolve_pyproject_toml() -> anyhow::Result<()> { ), python_version: None, python_platform: None, + universal: false, exclude_newer: Some( ExcludeNewer( 2024-03-25T00:00:00Z, @@ -986,6 +992,7 @@ fn resolve_index_url() -> anyhow::Result<()> { ), python_version: None, python_platform: None, + universal: false, exclude_newer: Some( ExcludeNewer( 2024-03-25T00:00:00Z, @@ -1162,6 +1169,7 @@ fn resolve_index_url() -> anyhow::Result<()> { ), python_version: None, python_platform: None, + universal: false, exclude_newer: Some( ExcludeNewer( 2024-03-25T00:00:00Z, @@ -1311,6 +1319,7 @@ fn resolve_find_links() -> anyhow::Result<()> { ), python_version: None, python_platform: None, + universal: false, exclude_newer: Some( ExcludeNewer( 2024-03-25T00:00:00Z, @@ -1441,6 +1450,7 @@ fn resolve_top_level() -> anyhow::Result<()> { ), python_version: None, python_platform: None, + universal: false, exclude_newer: Some( ExcludeNewer( 2024-03-25T00:00:00Z, @@ -1609,6 +1619,7 @@ fn resolve_top_level() -> anyhow::Result<()> { ), python_version: None, python_platform: None, + universal: false, exclude_newer: Some( ExcludeNewer( 2024-03-25T00:00:00Z, @@ -1760,6 +1771,7 @@ fn resolve_top_level() -> anyhow::Result<()> { ), python_version: None, python_platform: None, + universal: false, exclude_newer: Some( ExcludeNewer( 2024-03-25T00:00:00Z, @@ -1890,6 +1902,7 @@ fn resolve_user_configuration() -> anyhow::Result<()> { ), python_version: None, python_platform: None, + universal: false, exclude_newer: Some( ExcludeNewer( 2024-03-25T00:00:00Z, @@ -2003,6 +2016,7 @@ fn resolve_user_configuration() -> anyhow::Result<()> { ), python_version: None, python_platform: None, + universal: false, exclude_newer: Some( ExcludeNewer( 2024-03-25T00:00:00Z, @@ -2116,6 +2130,7 @@ fn resolve_user_configuration() -> anyhow::Result<()> { ), python_version: None, python_platform: None, + universal: false, exclude_newer: Some( ExcludeNewer( 2024-03-25T00:00:00Z, @@ -2231,6 +2246,7 @@ fn resolve_user_configuration() -> anyhow::Result<()> { ), python_version: None, python_platform: None, + universal: false, exclude_newer: Some( ExcludeNewer( 2024-03-25T00:00:00Z, @@ -2371,6 +2387,7 @@ fn resolve_poetry_toml() -> anyhow::Result<()> { ), python_version: None, python_platform: None, + universal: false, exclude_newer: Some( ExcludeNewer( 2024-03-25T00:00:00Z, diff --git a/uv.schema.json b/uv.schema.json index 719a1d4b9157..041fb5cb9f13 100644 --- a/uv.schema.json +++ b/uv.schema.json @@ -784,6 +784,12 @@ "null" ] }, + "universal": { + "type": [ + "boolean", + "null" + ] + }, "upgrade": { "type": [ "boolean",