diff --git a/crates/pypi-types/src/metadata.rs b/crates/pypi-types/src/metadata.rs index 3aa24add303b..f1fb9c4a6c47 100644 --- a/crates/pypi-types/src/metadata.rs +++ b/crates/pypi-types/src/metadata.rs @@ -1,5 +1,6 @@ //! Derived from `pypi_types_crate`. +use std::io::BufRead; use std::str::FromStr; use indexmap::IndexMap; @@ -10,7 +11,8 @@ use thiserror::Error; use tracing::warn; use pep440_rs::{Version, VersionParseError, VersionSpecifiers, VersionSpecifiersParseError}; -use pep508_rs::{Pep508Error, Requirement}; +use pep508_rs::marker::MarkerValueExtra; +use pep508_rs::{ExtraOperator, MarkerExpression, MarkerTree, Pep508Error, Requirement}; use uv_normalize::{ExtraName, InvalidNameError, PackageName}; use crate::lenient_requirement::LenientRequirement; @@ -62,6 +64,8 @@ pub enum MetadataError { DynamicField(&'static str), #[error("The project uses Poetry's syntax to declare its dependencies, despite including a `project` table in `pyproject.toml`")] PoetrySyntax, + #[error("Failed to read `requires.txt` contents")] + RequiresTxtContents(#[from] std::io::Error), } impl From> for MetadataError { @@ -492,6 +496,109 @@ impl RequiresDist { } } +/// `requires.txt` metadata as defined in . +/// +/// This is a subset of the full metadata specification, and only includes the fields that are +/// included in the legacy `requires.txt` file. +#[derive(Deserialize, Debug, Clone)] +#[serde(rename_all = "kebab-case")] +pub struct RequiresTxt { + pub requires_dist: Vec>, + pub provides_extras: Vec, +} + +impl RequiresTxt { + /// Parse the [`RequiresTxt`] from a `requires.txt` file, as included in an `egg-info`. + /// + /// See: + pub fn parse(content: &[u8]) -> Result { + let mut requires_dist = vec![]; + let mut provides_extras = vec![]; + let mut current_marker = MarkerTree::default(); + + for line in content.lines() { + let line = line.map_err(MetadataError::RequiresTxtContents)?; + + let line = line.trim(); + if line.is_empty() { + continue; + } + + // When encountering a new section, parse the extra and marker from the header, e.g., + // `[:sys_platform == "win32"]` or `[dev]`. + if line.starts_with('[') { + let line = line.trim_start_matches('[').trim_end_matches(']'); + + // Split into extra and marker, both of which can be empty. + let (extra, marker) = { + let (extra, marker) = match line.split_once(':') { + Some((extra, marker)) => (Some(extra), Some(marker)), + None => (Some(line), None), + }; + let extra = extra.filter(|extra| !extra.is_empty()); + let marker = marker.filter(|marker| !marker.is_empty()); + (extra, marker) + }; + + // Parse the extra. + let extra = if let Some(extra) = extra { + if let Ok(extra) = ExtraName::from_str(extra) { + provides_extras.push(extra.clone()); + Some(MarkerValueExtra::Extra(extra)) + } else { + Some(MarkerValueExtra::Arbitrary(extra.to_string())) + } + } else { + None + }; + + // Parse the marker. + let marker = marker.map(MarkerTree::parse_str).transpose()?; + + // Create the marker tree. + match (extra, marker) { + (Some(extra), Some(mut marker)) => { + marker.and(MarkerTree::expression(MarkerExpression::Extra { + operator: ExtraOperator::Equal, + name: extra, + })); + current_marker = marker; + } + (Some(extra), None) => { + current_marker = MarkerTree::expression(MarkerExpression::Extra { + operator: ExtraOperator::Equal, + name: extra, + }); + } + (None, Some(marker)) => { + current_marker = marker; + } + (None, None) => { + current_marker = MarkerTree::default(); + } + } + + continue; + } + + // Parse the requirement. + let requirement = + Requirement::::from(LenientRequirement::from_str(line)?); + + // Add the markers and extra, if necessary. + requires_dist.push(Requirement { + marker: current_marker.clone(), + ..requirement + }); + } + + Ok(Self { + requires_dist, + provides_extras, + }) + } +} + /// The headers of a distribution metadata file. #[derive(Debug)] struct Headers<'a>(Vec>); @@ -531,7 +638,7 @@ mod tests { use pep440_rs::Version; use uv_normalize::PackageName; - use crate::MetadataError; + use crate::{MetadataError, RequiresTxt}; use super::Metadata23; @@ -677,4 +784,59 @@ mod tests { ); assert_eq!(meta.provides_extras, vec!["dotenv".parse().unwrap()]); } + + #[test] + fn test_requires_txt() { + let s = r" +Werkzeug>=0.14 +Jinja2>=2.10 + +[dev] +pytest>=3 +sphinx + +[dotenv] +python-dotenv + "; + let meta = RequiresTxt::parse(s.as_bytes()).unwrap(); + assert_eq!( + meta.requires_dist, + vec![ + "Werkzeug>=0.14".parse().unwrap(), + "Jinja2>=2.10".parse().unwrap(), + "pytest>=3; extra == \"dev\"".parse().unwrap(), + "sphinx; extra == \"dev\"".parse().unwrap(), + "python-dotenv; extra == \"dotenv\"".parse().unwrap(), + ] + ); + + let s = r" +Werkzeug>=0.14 + +[dev:] +Jinja2>=2.10 + +[:sys_platform == 'win32'] +pytest>=3 + +[] +sphinx + +[dotenv:sys_platform == 'darwin'] +python-dotenv + "; + let meta = RequiresTxt::parse(s.as_bytes()).unwrap(); + assert_eq!( + meta.requires_dist, + vec![ + "Werkzeug>=0.14".parse().unwrap(), + "Jinja2>=2.10 ; extra == \"dev\"".parse().unwrap(), + "pytest>=3; sys_platform == 'win32'".parse().unwrap(), + "sphinx".parse().unwrap(), + "python-dotenv; sys_platform == 'darwin' and extra == \"dotenv\"" + .parse() + .unwrap(), + ] + ); + } } diff --git a/crates/uv-distribution/src/error.rs b/crates/uv-distribution/src/error.rs index 2a05ddc92f92..f724c5833c09 100644 --- a/crates/uv-distribution/src/error.rs +++ b/crates/uv-distribution/src/error.rs @@ -71,8 +71,14 @@ pub enum Error { Extract(#[from] uv_extract::Error), #[error("The source distribution is missing a `PKG-INFO` file")] MissingPkgInfo, + #[error("The source distribution is missing an `egg-info` directory")] + MissingEggInfo, + #[error("The source distribution is missing a `requires.txt` file")] + MissingRequiresTxt, #[error("Failed to extract static metadata from `PKG-INFO`")] PkgInfo(#[source] pypi_types::MetadataError), + #[error("Failed to extract metadata from `requires.txt`")] + RequiresTxt(#[source] pypi_types::MetadataError), #[error("The source distribution is missing a `pyproject.toml` file")] MissingPyprojectToml, #[error("Failed to extract static metadata from `pyproject.toml`")] diff --git a/crates/uv-distribution/src/source/mod.rs b/crates/uv-distribution/src/source/mod.rs index e16b9103fa6e..6d4a36ea68c4 100644 --- a/crates/uv-distribution/src/source/mod.rs +++ b/crates/uv-distribution/src/source/mod.rs @@ -20,7 +20,7 @@ use distribution_types::{ }; use install_wheel_rs::metadata::read_archive_metadata; use platform_tags::Tags; -use pypi_types::{HashDigest, Metadata23}; +use pypi_types::{HashDigest, Metadata12, Metadata23, RequiresTxt}; use uv_cache::{ ArchiveTimestamp, CacheBucket, CacheEntry, CacheShard, CachedByTimestamp, Timestamp, WheelCache, }; @@ -1505,6 +1505,30 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { source_root: &Path, subdirectory: Option<&Path>, ) -> Result, Error> { + // Attempt to read static metadata from the `pyproject.toml`. + match read_pyproject_toml(source_root, subdirectory).await { + Ok(metadata) => { + debug!("Found static `pyproject.toml` for: {source}"); + + // Validate the metadata. + validate(source, &metadata)?; + + return Ok(Some(metadata)); + } + Err( + err @ (Error::MissingPyprojectToml + | Error::PyprojectToml( + pypi_types::MetadataError::Pep508Error(_) + | pypi_types::MetadataError::DynamicField(_) + | pypi_types::MetadataError::FieldNotFound(_) + | pypi_types::MetadataError::PoetrySyntax, + )), + ) => { + debug!("No static `pyproject.toml` available for: {source} ({err:?})"); + } + Err(err) => return Err(err), + } + // Attempt to read static metadata from the `PKG-INFO` file. match read_pkg_info(source_root, subdirectory).await { Ok(metadata) => { @@ -1521,8 +1545,7 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { pypi_types::MetadataError::Pep508Error(_) | pypi_types::MetadataError::DynamicField(_) | pypi_types::MetadataError::FieldNotFound(_) - | pypi_types::MetadataError::UnsupportedMetadataVersion(_) - | pypi_types::MetadataError::PoetrySyntax, + | pypi_types::MetadataError::UnsupportedMetadataVersion(_), )), ) => { debug!("No static `PKG-INFO` available for: {source} ({err:?})"); @@ -1530,10 +1553,10 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { Err(err) => return Err(err), } - // Attempt to read static metadata from the `pyproject.toml`. - match read_pyproject_toml(source_root, subdirectory).await { + // Attempt to read static metadata from the `egg-info` directory. + match read_egg_info(source_root, subdirectory).await { Ok(metadata) => { - debug!("Found static `pyproject.toml` for: {source}"); + debug!("Found static `egg-info` for: {source}"); // Validate the metadata. validate(source, &metadata)?; @@ -1541,16 +1564,21 @@ impl<'a, T: BuildContext> SourceDistributionBuilder<'a, T> { return Ok(Some(metadata)); } Err( - err @ (Error::MissingPyprojectToml - | Error::PyprojectToml( + err @ (Error::MissingEggInfo + | Error::MissingRequiresTxt + | Error::MissingPkgInfo + | Error::RequiresTxt( + pypi_types::MetadataError::Pep508Error(_) + | pypi_types::MetadataError::RequiresTxtContents(_), + ) + | Error::PkgInfo( pypi_types::MetadataError::Pep508Error(_) | pypi_types::MetadataError::DynamicField(_) | pypi_types::MetadataError::FieldNotFound(_) - | pypi_types::MetadataError::UnsupportedMetadataVersion(_) - | pypi_types::MetadataError::PoetrySyntax, + | pypi_types::MetadataError::UnsupportedMetadataVersion(_), )), ) => { - debug!("No static `pyproject.toml` available for: {source} ({err:?})"); + debug!("No static `egg-info` available for: {source} ({err:?})"); } Err(err) => return Err(err), } @@ -1667,6 +1695,105 @@ impl LocalRevisionPointer { } } +/// Read the [`Metadata23`] by combining a source distribution's `PKG-INFO` file with a +/// `requires.txt`. +/// +/// `requires.txt` is a legacy concept from setuptools. For example, here's +/// `Flask.egg-info/requires.txt` from Flask's 1.0 release: +/// +/// ```txt +/// Werkzeug>=0.14 +/// Jinja2>=2.10 +/// itsdangerous>=0.24 +/// click>=5.1 +/// +/// [dev] +/// pytest>=3 +/// coverage +/// tox +/// sphinx +/// pallets-sphinx-themes +/// sphinxcontrib-log-cabinet +/// +/// [docs] +/// sphinx +/// pallets-sphinx-themes +/// sphinxcontrib-log-cabinet +/// +/// [dotenv] +/// python-dotenv +/// ``` +/// +/// See: +async fn read_egg_info( + source_tree: &Path, + subdirectory: Option<&Path>, +) -> Result { + fn find_egg_info(source_tree: &Path) -> std::io::Result> { + for entry in fs_err::read_dir(source_tree)? { + let entry = entry?; + let ty = entry.file_type()?; + if ty.is_dir() { + let path = entry.path(); + if path + .extension() + .is_some_and(|ext| ext.eq_ignore_ascii_case("egg-info")) + { + return Ok(Some(path)); + } + } + } + Ok(None) + } + + let directory = match subdirectory { + Some(subdirectory) => Cow::Owned(source_tree.join(subdirectory)), + None => Cow::Borrowed(source_tree), + }; + + // Locate the `egg-info` directory. + let egg_info = match find_egg_info(directory.as_ref()) { + Ok(Some(path)) => path, + Ok(None) => return Err(Error::MissingEggInfo), + Err(err) => return Err(Error::CacheRead(err)), + }; + + // Read the `requires.txt`. + let requires_txt = egg_info.join("requires.txt"); + let content = match fs::read(requires_txt).await { + Ok(content) => content, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + return Err(Error::MissingRequiresTxt); + } + Err(err) => return Err(Error::CacheRead(err)), + }; + + // Parse the `requires.txt. + let requires_txt = RequiresTxt::parse(&content).map_err(Error::RequiresTxt)?; + + // Read the `PKG-INFO` file. + let pkg_info = egg_info.join("PKG-INFO"); + let content = match fs::read(pkg_info).await { + Ok(content) => content, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + return Err(Error::MissingPkgInfo); + } + Err(err) => return Err(Error::CacheRead(err)), + }; + + // Parse the metadata. + let metadata = Metadata12::parse_metadata(&content).map_err(Error::PkgInfo)?; + + // Combine the sources. + Ok(Metadata23 { + name: metadata.name, + version: metadata.version, + requires_python: metadata.requires_python, + requires_dist: requires_txt.requires_dist, + provides_extras: requires_txt.provides_extras, + }) +} + /// Read the [`Metadata23`] from a source distribution's `PKG-INFO` file, if it uses Metadata 2.2 /// or later _and_ none of the required fields (`Requires-Python`, `Requires-Dist`, and /// `Provides-Extra`) are marked as dynamic. diff --git a/crates/uv/tests/pip_compile.rs b/crates/uv/tests/pip_compile.rs index 4713b8e424c0..f89c6f1a5e04 100644 --- a/crates/uv/tests/pip_compile.rs +++ b/crates/uv/tests/pip_compile.rs @@ -11995,3 +11995,41 @@ fn universal_constrained_environment() -> Result<()> { Ok(()) } + +/// Resolve a version of Flask that ships a `requires.txt` file in an `egg-info` directory, but +/// otherwise doesn't include static metadata. +#[test] +fn compile_requires_txt() -> Result<()> { + let context = TestContext::new("3.12"); + let requirements_in = context.temp_dir.child("requirements.in"); + requirements_in.write_str("flask @ https://files.pythonhosted.org/packages/36/70/2234ee8842148cef44261c2cebca3a6384894bce6112b73b18693cdcc62f/Flask-1.0.4.tar.gz")?; + + uv_snapshot!(context + .pip_compile() + .arg("requirements.in"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + # This file was autogenerated by uv via the following command: + # uv pip compile --cache-dir [CACHE_DIR] requirements.in + click==8.1.7 + # via flask + flask @ https://files.pythonhosted.org/packages/36/70/2234ee8842148cef44261c2cebca3a6384894bce6112b73b18693cdcc62f/Flask-1.0.4.tar.gz + # via -r requirements.in + itsdangerous==2.1.2 + # via flask + jinja2==3.1.3 + # via flask + markupsafe==2.1.5 + # via + # jinja2 + # werkzeug + werkzeug==3.0.1 + # via flask + + ----- stderr ----- + Resolved 6 packages in [TIME] + "###); + + Ok(()) +}