From ed91b1d5627871d55ea53838e651a750ff92f2e0 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Thu, 16 May 2024 17:12:52 -0400 Subject: [PATCH] Parse and store extras on editables (#3629) ## Summary As a follow-up to #3622, we now parse and store (but don't respect) markers on editable requirements. --- crates/pep508-rs/src/lib.rs | 6 +- crates/requirements-txt/src/lib.rs | 63 ++++++++++++++-- ...ts_txt__test__parse-unix-editable.txt.snap | 75 ++++++++++++++++++- ...txt__test__parse-windows-editable.txt.snap | 75 ++++++++++++++++++- .../test-data/requirements-txt/editable.txt | 8 +- crates/uv-requirements/src/specification.rs | 1 + crates/uv/src/commands/pip/compile.rs | 1 + crates/uv/src/commands/pip/editables.rs | 1 + 8 files changed, 211 insertions(+), 19 deletions(-) diff --git a/crates/pep508-rs/src/lib.rs b/crates/pep508-rs/src/lib.rs index f88e5233bf1c..99065fc17e4d 100644 --- a/crates/pep508-rs/src/lib.rs +++ b/crates/pep508-rs/src/lib.rs @@ -137,7 +137,7 @@ create_exception!( /// A PEP 508 dependency specifier. #[derive(Hash, Debug, Clone, Eq, PartialEq)] pub struct Requirement { - /// The distribution name such as `numpy` in + /// The distribution name such as `requests` in /// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`. pub name: PackageName, /// The list of extras such as `security`, `tests` in @@ -145,7 +145,7 @@ pub struct Requirement { pub extras: Vec, /// The version specifier such as `>= 2.8.1`, `== 2.8.*` in /// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`. - /// or a url + /// or a URL. pub version_or_url: Option>, /// The markers such as `python_version > "3.8"` in /// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`. @@ -241,7 +241,7 @@ impl Deref for PyRequirement { #[cfg(feature = "pyo3")] #[pymethods] impl PyRequirement { - /// The distribution name such as `numpy` in + /// The distribution name such as `requests` in /// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"` #[getter] pub fn name(&self) -> String { diff --git a/crates/requirements-txt/src/lib.rs b/crates/requirements-txt/src/lib.rs index 82ffb4e8d876..441a6bf9b71e 100644 --- a/crates/requirements-txt/src/lib.rs +++ b/crates/requirements-txt/src/lib.rs @@ -48,7 +48,7 @@ use distribution_types::{ ParsedUrlError, Requirement, UnresolvedRequirement, UnresolvedRequirementSpecification, }; use pep508_rs::{ - expand_env_vars, split_scheme, strip_host, Extras, Pep508Error, Pep508ErrorSource, + expand_env_vars, split_scheme, strip_host, Extras, MarkerTree, Pep508Error, Pep508ErrorSource, RequirementOrigin, Scheme, VerbatimUrl, }; #[cfg(feature = "http")] @@ -168,6 +168,8 @@ pub struct EditableRequirement { pub url: VerbatimUrl, /// The extras that should be included when resolving the editable requirements. pub extras: Vec, + /// The markers such as `python_version > "3.8"` in `-e ../editable ; python_version > "3.8"`. + pub marker: Option, /// The local path to the editable. pub path: PathBuf, /// The source file containing the requirement. @@ -199,10 +201,37 @@ impl EditableRequirement { origin: Option<&Path>, working_dir: impl AsRef, ) -> Result { - // Identify (but discard) any markers, to match pip. - let given = Self::split_markers(given) - .map(|(requirement, _)| requirement) - .unwrap_or(given); + // Identify the markers. + let (given, marker) = if let Some((requirement, marker)) = Self::split_markers(given) { + let marker = MarkerTree::from_str(marker).map_err(|err| { + // Map from error on the markers to error on the whole requirement. + let err = Pep508Error { + message: err.message, + start: requirement.len() + err.start, + len: err.len, + input: given.to_string(), + }; + match err.message { + Pep508ErrorSource::String(_) | Pep508ErrorSource::UrlError(_) => { + RequirementsTxtParserError::Pep508 { + start: err.start, + end: err.start + err.len, + source: err, + } + } + Pep508ErrorSource::UnsupportedRequirement(_) => { + RequirementsTxtParserError::UnsupportedRequirement { + start: err.start, + end: err.start + err.len, + source: err, + } + } + } + })?; + (requirement, Some(marker)) + } else { + (given, None) + }; // Identify the extras. let (requirement, extras) = if let Some((requirement, extras)) = Self::split_extras(given) { @@ -279,6 +308,7 @@ impl EditableRequirement { Ok(Self { url, extras, + marker, path, origin: origin.map(Path::to_path_buf).map(RequirementOrigin::File), }) @@ -322,7 +352,27 @@ impl EditableRequirement { } else if c == ']' { depth -= 1; } else if depth == 0 && c.is_whitespace() { - return Some(given.split_at(index)); + // We found the end of the requirement; now, find the start of the markers, + // delimited by a semicolon. + let (requirement, markers) = given.split_at(index); + + // Skip the whitespace. + for (index, c) in markers.char_indices() { + if backslash { + backslash = false; + } else if c == '\\' { + backslash = true; + } else if c.is_whitespace() { + continue; + } else if c == ';' { + // The marker starts just after the semicolon. + let markers = &markers[index + 1..]; + return Some((requirement, markers)); + } else { + // We saw some other character, so this isn't a marker. + return None; + } + } } } None @@ -1995,6 +2045,7 @@ mod test { ), }, extras: [], + marker: None, path: "/foo/bar", origin: Some( File( diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-unix-editable.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-unix-editable.txt.snap index 11ec84230b76..2df2e5ecc879 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-unix-editable.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-unix-editable.txt.snap @@ -31,6 +31,7 @@ RequirementsTxt { "dev", ), ], + marker: None, path: "/editable", origin: Some( File( @@ -63,6 +64,7 @@ RequirementsTxt { "dev", ), ], + marker: None, path: "/editable", origin: Some( File( @@ -95,6 +97,28 @@ RequirementsTxt { "dev", ), ], + marker: Some( + And( + [ + Expression( + Version { + key: PythonVersion, + specifier: VersionSpecifier { + operator: GreaterThanEqual, + version: "3.9", + }, + }, + ), + Expression( + String { + key: OsName, + operator: Equal, + value: "posix", + }, + ), + ], + ), + ), path: "/editable", origin: Some( File( @@ -127,6 +151,28 @@ RequirementsTxt { "dev", ), ], + marker: Some( + And( + [ + Expression( + Version { + key: PythonVersion, + specifier: VersionSpecifier { + operator: GreaterThanEqual, + version: "3.9", + }, + }, + ), + Expression( + String { + key: OsName, + operator: Equal, + value: "posix", + }, + ), + ], + ), + ), path: "/editable", origin: Some( File( @@ -152,6 +198,28 @@ RequirementsTxt { ), }, extras: [], + marker: Some( + And( + [ + Expression( + Version { + key: PythonVersion, + specifier: VersionSpecifier { + operator: GreaterThanEqual, + version: "3.9", + }, + }, + ), + Expression( + String { + key: OsName, + operator: Equal, + value: "posix", + }, + ), + ], + ), + ), path: "/editable", origin: Some( File( @@ -168,16 +236,17 @@ RequirementsTxt { password: None, host: None, port: None, - path: "/editable;", + path: "/editable;%20python_version%20%3E=%20%223.9%22%20and%20os_name%20==%20%22posix%22", query: None, fragment: None, }, given: Some( - "./editable;", + "./editable; python_version >= \"3.9\" and os_name == \"posix\"", ), }, extras: [], - path: "/editable;", + marker: None, + path: "/editable; python_version >= \"3.9\" and os_name == \"posix\"", origin: Some( File( "/editable.txt", diff --git a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-windows-editable.txt.snap b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-windows-editable.txt.snap index 16fdf73553e4..ca4a73218048 100644 --- a/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-windows-editable.txt.snap +++ b/crates/requirements-txt/src/snapshots/requirements_txt__test__parse-windows-editable.txt.snap @@ -31,6 +31,7 @@ RequirementsTxt { "dev", ), ], + marker: None, path: "/editable", origin: Some( File( @@ -63,6 +64,7 @@ RequirementsTxt { "dev", ), ], + marker: None, path: "/editable", origin: Some( File( @@ -95,6 +97,28 @@ RequirementsTxt { "dev", ), ], + marker: Some( + And( + [ + Expression( + Version { + key: PythonVersion, + specifier: VersionSpecifier { + operator: GreaterThanEqual, + version: "3.9", + }, + }, + ), + Expression( + String { + key: OsName, + operator: Equal, + value: "posix", + }, + ), + ], + ), + ), path: "/editable", origin: Some( File( @@ -127,6 +151,28 @@ RequirementsTxt { "dev", ), ], + marker: Some( + And( + [ + Expression( + Version { + key: PythonVersion, + specifier: VersionSpecifier { + operator: GreaterThanEqual, + version: "3.9", + }, + }, + ), + Expression( + String { + key: OsName, + operator: Equal, + value: "posix", + }, + ), + ], + ), + ), path: "/editable", origin: Some( File( @@ -152,6 +198,28 @@ RequirementsTxt { ), }, extras: [], + marker: Some( + And( + [ + Expression( + Version { + key: PythonVersion, + specifier: VersionSpecifier { + operator: GreaterThanEqual, + version: "3.9", + }, + }, + ), + Expression( + String { + key: OsName, + operator: Equal, + value: "posix", + }, + ), + ], + ), + ), path: "/editable", origin: Some( File( @@ -168,16 +236,17 @@ RequirementsTxt { password: None, host: None, port: None, - path: "//editable;", + path: "//editable;%20python_version%20%3E=%20%223.9%22%20and%20os_name%20==%20%22posix%22", query: None, fragment: None, }, given: Some( - "./editable;", + "./editable; python_version >= \"3.9\" and os_name == \"posix\"", ), }, extras: [], - path: "/editable;", + marker: None, + path: "/editable; python_version >= \"3.9\" and os_name == \"posix\"", origin: Some( File( "/editable.txt", diff --git a/crates/requirements-txt/test-data/requirements-txt/editable.txt b/crates/requirements-txt/test-data/requirements-txt/editable.txt index eb059486c1c9..c28864116467 100644 --- a/crates/requirements-txt/test-data/requirements-txt/editable.txt +++ b/crates/requirements-txt/test-data/requirements-txt/editable.txt @@ -5,13 +5,13 @@ -e ./editable[d, dev] # OK --e ./editable[d,dev] ; python_version >= "3.9" and python_ver +-e ./editable[d,dev] ; python_version >= "3.9" and os_name == "posix" # OK (whitespace between extras; disallowed by pip) --e ./editable[d, dev] ; python_version >= "3.9" and python_ver +-e ./editable[d, dev] ; python_version >= "3.9" and os_name == "posix" # OK --e ./editable ; python_version >= "3.9" and python_ver +-e ./editable ; python_version >= "3.9" and os_name == "posix" # Disallowed (missing whitespace before colon) --e ./editable; python_version >= "3.9" and python_ver +-e ./editable; python_version >= "3.9" and os_name == "posix" diff --git a/crates/uv-requirements/src/specification.rs b/crates/uv-requirements/src/specification.rs index 811d2e1a7b6b..e3fc5a7e3107 100644 --- a/crates/uv-requirements/src/specification.rs +++ b/crates/uv-requirements/src/specification.rs @@ -191,6 +191,7 @@ impl RequirementsSpecification { Either::Left(EditableRequirement { url, path, + marker: requirement.marker, extras: requirement.extras, origin: requirement.origin, }) diff --git a/crates/uv/src/commands/pip/compile.rs b/crates/uv/src/commands/pip/compile.rs index 2080543027e9..f786b03fb886 100644 --- a/crates/uv/src/commands/pip/compile.rs +++ b/crates/uv/src/commands/pip/compile.rs @@ -427,6 +427,7 @@ pub(crate) async fn pip_compile( url, extras, path, + marker: _, origin: _, } = editable; LocalEditable { url, path, extras } diff --git a/crates/uv/src/commands/pip/editables.rs b/crates/uv/src/commands/pip/editables.rs index efaade03bc5a..a4b385f7aac5 100644 --- a/crates/uv/src/commands/pip/editables.rs +++ b/crates/uv/src/commands/pip/editables.rs @@ -108,6 +108,7 @@ impl ResolvedEditables { url, path, extras, + marker: _, origin: _, } = editable; LocalEditable {