Skip to content

Commit

Permalink
Parse and store extras on editables (#3629)
Browse files Browse the repository at this point in the history 
## Summary

As a follow-up to #3622, we now parse and store (but don't respect)
markers on editable requirements.
  • Loading branch information
@charliermarsh
charliermarsh authored May 16, 2024
1 parent 7f73f7b commit ed91b1d
Show file tree
Hide file tree
Showing 8 changed files with 211 additions and 19 deletions.
6 changes: 3 additions & 3 deletions crates/pep508-rs/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,15 +137,15 @@ create_exception!(
/// A PEP 508 dependency specifier.
#[derive(Hash, Debug, Clone, Eq, PartialEq)]
pub struct Requirement<T: Pep508Url = VerbatimUrl> {
/// The distribution name such as `numpy` in
/// The distribution name such as `requests` in
/// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`.
pub name: PackageName,
/// The list of extras such as `security`, `tests` in
/// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`.
pub extras: Vec<ExtraName>,
/// The version specifier such as `>= 2.8.1`, `== 2.8.*` in
/// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`.
/// or a url
/// or a URL.
pub version_or_url: Option<VersionOrUrl<T>>,
/// The markers such as `python_version > "3.8"` in
/// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`.
Expand Down Expand Up @@ -241,7 +241,7 @@ impl Deref for PyRequirement {
#[cfg(feature = "pyo3")]
#[pymethods]
impl PyRequirement {
/// The distribution name such as `numpy` in
/// The distribution name such as `requests` in
/// `requests [security,tests] >= 2.8.1, == 2.8.* ; python_version > "3.8"`
#[getter]
pub fn name(&self) -> String {
Expand Down
63 changes: 57 additions & 6 deletions crates/requirements-txt/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ use distribution_types::{
ParsedUrlError, Requirement, UnresolvedRequirement, UnresolvedRequirementSpecification,
};
use pep508_rs::{
expand_env_vars, split_scheme, strip_host, Extras, Pep508Error, Pep508ErrorSource,
expand_env_vars, split_scheme, strip_host, Extras, MarkerTree, Pep508Error, Pep508ErrorSource,
RequirementOrigin, Scheme, VerbatimUrl,
};
#[cfg(feature = "http")]
Expand Down Expand Up @@ -168,6 +168,8 @@ pub struct EditableRequirement {
pub url: VerbatimUrl,
/// The extras that should be included when resolving the editable requirements.
pub extras: Vec<ExtraName>,
/// The markers such as `python_version > "3.8"` in `-e ../editable ; python_version > "3.8"`.
pub marker: Option<MarkerTree>,
/// The local path to the editable.
pub path: PathBuf,
/// The source file containing the requirement.
Expand Down Expand Up @@ -199,10 +201,37 @@ impl EditableRequirement {
origin: Option<&Path>,
working_dir: impl AsRef<Path>,
) -> Result<Self, RequirementsTxtParserError> {
// Identify (but discard) any markers, to match pip.
let given = Self::split_markers(given)
.map(|(requirement, _)| requirement)
.unwrap_or(given);
// Identify the markers.
let (given, marker) = if let Some((requirement, marker)) = Self::split_markers(given) {
let marker = MarkerTree::from_str(marker).map_err(|err| {
// Map from error on the markers to error on the whole requirement.
let err = Pep508Error {
message: err.message,
start: requirement.len() + err.start,
len: err.len,
input: given.to_string(),
};
match err.message {
Pep508ErrorSource::String(_) | Pep508ErrorSource::UrlError(_) => {
RequirementsTxtParserError::Pep508 {
start: err.start,
end: err.start + err.len,
source: err,
}
}
Pep508ErrorSource::UnsupportedRequirement(_) => {
RequirementsTxtParserError::UnsupportedRequirement {
start: err.start,
end: err.start + err.len,
source: err,
}
}
}
})?;
(requirement, Some(marker))
} else {
(given, None)
};

// Identify the extras.
let (requirement, extras) = if let Some((requirement, extras)) = Self::split_extras(given) {
Expand Down Expand Up @@ -279,6 +308,7 @@ impl EditableRequirement {
Ok(Self {
url,
extras,
marker,
path,
origin: origin.map(Path::to_path_buf).map(RequirementOrigin::File),
})
Expand Down Expand Up @@ -322,7 +352,27 @@ impl EditableRequirement {
} else if c == ']' {
depth -= 1;
} else if depth == 0 && c.is_whitespace() {
return Some(given.split_at(index));
// We found the end of the requirement; now, find the start of the markers,
// delimited by a semicolon.
let (requirement, markers) = given.split_at(index);

// Skip the whitespace.
for (index, c) in markers.char_indices() {
if backslash {
backslash = false;
} else if c == '\\' {
backslash = true;
} else if c.is_whitespace() {
continue;
} else if c == ';' {
// The marker starts just after the semicolon.
let markers = &markers[index + 1..];
return Some((requirement, markers));
} else {
// We saw some other character, so this isn't a marker.
return None;
}
}
}
}
None
Expand Down Expand Up @@ -1995,6 +2045,7 @@ mod test {
),
},
extras: [],
marker: None,
path: "/foo/bar",
origin: Some(
File(
Expand Down
75 changes: 72 additions & 3 deletions crates/requirements-txt/src/snapshots/requirements_txt__test__parse-unix-editable.txt.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ RequirementsTxt {
"dev",
),
],
marker: None,
path: "<REQUIREMENTS_DIR>/editable",
origin: Some(
File(
Expand Down Expand Up @@ -63,6 +64,7 @@ RequirementsTxt {
"dev",
),
],
marker: None,
path: "<REQUIREMENTS_DIR>/editable",
origin: Some(
File(
Expand Down Expand Up @@ -95,6 +97,28 @@ RequirementsTxt {
"dev",
),
],
marker: Some(
And(
[
Expression(
Version {
key: PythonVersion,
specifier: VersionSpecifier {
operator: GreaterThanEqual,
version: "3.9",
},
},
),
Expression(
String {
key: OsName,
operator: Equal,
value: "posix",
},
),
],
),
),
path: "<REQUIREMENTS_DIR>/editable",
origin: Some(
File(
Expand Down Expand Up @@ -127,6 +151,28 @@ RequirementsTxt {
"dev",
),
],
marker: Some(
And(
[
Expression(
Version {
key: PythonVersion,
specifier: VersionSpecifier {
operator: GreaterThanEqual,
version: "3.9",
},
},
),
Expression(
String {
key: OsName,
operator: Equal,
value: "posix",
},
),
],
),
),
path: "<REQUIREMENTS_DIR>/editable",
origin: Some(
File(
Expand All @@ -152,6 +198,28 @@ RequirementsTxt {
),
},
extras: [],
marker: Some(
And(
[
Expression(
Version {
key: PythonVersion,
specifier: VersionSpecifier {
operator: GreaterThanEqual,
version: "3.9",
},
},
),
Expression(
String {
key: OsName,
operator: Equal,
value: "posix",
},
),
],
),
),
path: "<REQUIREMENTS_DIR>/editable",
origin: Some(
File(
Expand All @@ -168,16 +236,17 @@ RequirementsTxt {
password: None,
host: None,
port: None,
path: "<REQUIREMENTS_DIR>/editable;",
path: "<REQUIREMENTS_DIR>/editable;%20python_version%20%3E=%20%223.9%22%20and%20os_name%20==%20%22posix%22",
query: None,
fragment: None,
},
given: Some(
"./editable;",
"./editable; python_version >= \"3.9\" and os_name == \"posix\"",
),
},
extras: [],
path: "<REQUIREMENTS_DIR>/editable;",
marker: None,
path: "<REQUIREMENTS_DIR>/editable; python_version >= \"3.9\" and os_name == \"posix\"",
origin: Some(
File(
"<REQUIREMENTS_DIR>/editable.txt",
Expand Down
75 changes: 72 additions & 3 deletions ...es/requirements-txt/src/snapshots/requirements_txt__test__parse-windows-editable.txt.snap
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ RequirementsTxt {
"dev",
),
],
marker: None,
path: "<REQUIREMENTS_DIR>/editable",
origin: Some(
File(
Expand Down Expand Up @@ -63,6 +64,7 @@ RequirementsTxt {
"dev",
),
],
marker: None,
path: "<REQUIREMENTS_DIR>/editable",
origin: Some(
File(
Expand Down Expand Up @@ -95,6 +97,28 @@ RequirementsTxt {
"dev",
),
],
marker: Some(
And(
[
Expression(
Version {
key: PythonVersion,
specifier: VersionSpecifier {
operator: GreaterThanEqual,
version: "3.9",
},
},
),
Expression(
String {
key: OsName,
operator: Equal,
value: "posix",
},
),
],
),
),
path: "<REQUIREMENTS_DIR>/editable",
origin: Some(
File(
Expand Down Expand Up @@ -127,6 +151,28 @@ RequirementsTxt {
"dev",
),
],
marker: Some(
And(
[
Expression(
Version {
key: PythonVersion,
specifier: VersionSpecifier {
operator: GreaterThanEqual,
version: "3.9",
},
},
),
Expression(
String {
key: OsName,
operator: Equal,
value: "posix",
},
),
],
),
),
path: "<REQUIREMENTS_DIR>/editable",
origin: Some(
File(
Expand All @@ -152,6 +198,28 @@ RequirementsTxt {
),
},
extras: [],
marker: Some(
And(
[
Expression(
Version {
key: PythonVersion,
specifier: VersionSpecifier {
operator: GreaterThanEqual,
version: "3.9",
},
},
),
Expression(
String {
key: OsName,
operator: Equal,
value: "posix",
},
),
],
),
),
path: "<REQUIREMENTS_DIR>/editable",
origin: Some(
File(
Expand All @@ -168,16 +236,17 @@ RequirementsTxt {
password: None,
host: None,
port: None,
path: "/<REQUIREMENTS_DIR>/editable;",
path: "/<REQUIREMENTS_DIR>/editable;%20python_version%20%3E=%20%223.9%22%20and%20os_name%20==%20%22posix%22",
query: None,
fragment: None,
},
given: Some(
"./editable;",
"./editable; python_version >= \"3.9\" and os_name == \"posix\"",
),
},
extras: [],
path: "<REQUIREMENTS_DIR>/editable;",
marker: None,
path: "<REQUIREMENTS_DIR>/editable; python_version >= \"3.9\" and os_name == \"posix\"",
origin: Some(
File(
"<REQUIREMENTS_DIR>/editable.txt",
Expand Down
8 changes: 4 additions & 4 deletions crates/requirements-txt/test-data/requirements-txt/editable.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@
-e ./editable[d, dev]

# OK
-e ./editable[d,dev] ; python_version >= "3.9" and python_ver
-e ./editable[d,dev] ; python_version >= "3.9" and os_name == "posix"

# OK (whitespace between extras; disallowed by pip)
-e ./editable[d, dev] ; python_version >= "3.9" and python_ver
-e ./editable[d, dev] ; python_version >= "3.9" and os_name == "posix"

# OK
-e ./editable ; python_version >= "3.9" and python_ver
-e ./editable ; python_version >= "3.9" and os_name == "posix"

# Disallowed (missing whitespace before colon)
-e ./editable; python_version >= "3.9" and python_ver
-e ./editable; python_version >= "3.9" and os_name == "posix"
Loading

0 comments on commit ed91b1d

Please sign in to comment.