Skip to content

Commit

Permalink
Remove incompatible wheels from uv.lock
Browse files Browse the repository at this point in the history
Remove wheels that don't match the required python version from the lockfile. For example, we remove `charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl` when we have `requires-python = ">=3.12"`.

Our snapshots barely show changes since we avoid the large binaries for which matters. Here are 3 real world `uv.lock` before/after comparisons to show the large difference:
* [warehouse](https://gist.github.com/konstin/9a1ed6a32b410e250fcf4c6ea8c536a5) 5677 -> 4214
* [transformers](https://gist.github.com/konstin/5636281b5226f64aa44ce3244d5230cd) 6484 -> 5816
* [github-wikidata-bot](https://gist.github.com/konstin/ebbd7b9474523aaa61d9a8945bc02071) 793 -> 454

We only remove wheels we are certain don't match the python version and still keep those with unknown tags. We could remove even more wheels by also considering other markers, e.g. removing linux wheels for a windows-only dep, but we would trade complex, easy-to-get-wrong logic for diminishing returns.
  • Loading branch information
konstin committed Jul 4, 2024
1 parent 576ba9c commit 37c9b4e
Show file tree
Hide file tree
Showing 4 changed files with 96 additions and 187 deletions.
87 changes: 86 additions & 1 deletion crates/distribution-filename/src/wheel.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use serde::{de, Deserialize, Deserializer, Serialize, Serializer};
use thiserror::Error;
use url::Url;

use pep440_rs::{Version, VersionParseError};
use pep440_rs::{Version, VersionParseError, VersionSpecifiers};
use platform_tags::{TagCompatibility, Tags};
use uv_normalize::{InvalidNameError, PackageName};

Expand Down Expand Up @@ -60,6 +60,59 @@ impl WheelFilename {
compatible_tags.compatibility(&self.python_tag, &self.abi_tag, &self.platform_tag)
}

/// Returns `false` if the wheel's tags state it can't be used in the given Python version
/// range.
///
/// It is meant to filter out clearly unusable wheels with perfect specificity and acceptable
/// sensitivity, we return `true` if the tags are unknown.
pub fn matches_requires_python(&self, specifiers: &VersionSpecifiers) -> bool {
self.abi_tag.iter().any(|abi_tag| {
if abi_tag == "abi3" || abi_tag == "none" {
if self
.python_tag
.iter()
.all(|python_tag| python_tag.starts_with("py2"))
{
// Remove `py2-none-any` and `py27-none-any`.
false
} else {
// Universal tags are allowed.
true
}
} else if abi_tag.starts_with("cp2") || abi_tag.starts_with("pypy2") {
// Python 2 is never allowed.
false
} else if let Some(minor_no_dot_abi) = abi_tag.strip_prefix("cp3") {
// Remove ABI tags, both old (dmu) and future (t, and all other letters).
let minor_not_dot = minor_no_dot_abi.trim_matches(char::is_alphabetic);
let Ok(minor) = minor_not_dot.parse::<u64>() else {
// Unknown version pattern are allowed.
return true;
};

let version = Version::new([3, minor]);
specifiers.contains(&version)
} else if let Some(minor_no_dot_abi) = abi_tag.strip_prefix("pypy3") {
// Given `pypy39_pp73`, we just removed `pypy3`, now we remove `_pp73` ...
let Some((minor_not_dot, _)) = minor_no_dot_abi.split_once('_') else {
// Unknown version pattern are allowed.
return true;
};
// ... and get `9`.
let Ok(minor) = minor_not_dot.parse::<u64>() else {
// Unknown version pattern are allowed.
return true;
};

let version = Version::new([3, minor]);
specifiers.contains(&version)
} else {
// Unknown python tag -> allowed.
true
}
})
}

/// The wheel filename without the extension.
pub fn stem(&self) -> String {
format!(
Expand Down Expand Up @@ -332,4 +385,36 @@ mod tests {
);
}
}

#[test]
fn test_requires_python_included() {
let version_specifiers = VersionSpecifiers::from_str("==3.10.*").unwrap();
let wheel_names = &[
"bcrypt-4.1.3-cp37-abi3-macosx_10_12_universal2.whl",
"black-24.4.2-cp310-cp310-win_amd64.whl",
"cbor2-5.6.4-py3-none-any.whl",
"watchfiles-0.22.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl",
];
for wheel_name in wheel_names {
assert!(WheelFilename::from_str(wheel_name)
.unwrap()
.matches_requires_python(&version_specifiers));
}
}

#[test]
fn test_requires_python_dropped() {
let version_specifiers = VersionSpecifiers::from_str("==3.10.*").unwrap();
let wheel_names = &[
"PySocks-1.7.1-py27-none-any.whl",
"black-24.4.2-cp39-cp39-win_amd64.whl",
"psutil-6.0.0-cp36-cp36m-win32.whl",
"pydantic_core-2.20.1-pp39-pypy39_pp73-win_amd64.whl",
];
for wheel_name in wheel_names {
assert!(!WheelFilename::from_str(wheel_name)
.unwrap()
.matches_requires_python(&version_specifiers));
}
}
}
10 changes: 10 additions & 0 deletions crates/uv-resolver/src/lock.rs
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,16 @@ impl Lock {
}
}
}

// Remove wheels that don't match `requires-python` and can't be selected for
// installation.
if let Some(requires_python) = &requires_python {
dist.wheels.retain(|wheel| {
wheel
.filename
.matches_requires_python(requires_python.specifiers())
});
}
}
distributions.sort_by(|dist1, dist2| dist1.id.cmp(&dist2.id));

Expand Down
Loading

0 comments on commit 37c9b4e