Skip to content

Commit

Permalink
Add notes about hash generation
Browse files Browse the repository at this point in the history
Add basic hash

More notes

Looking at cache...
  • Loading branch information
charliermarsh committed Apr 9, 2024
1 parent 3aa2378 commit b1e9f80
Show file tree
Hide file tree
Showing 56 changed files with 2,593 additions and 465 deletions.
17 changes: 17 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,7 @@ indoc = { version = "2.0.4" }
itertools = { version = "0.12.1" }
junction = { version = "1.0.0" }
mailparse = { version = "0.14.0" }
md-5 = { version = "0.10.6" }
miette = { version = "7.2.0" }
nanoid = { version = "0.4.0" }
once_cell = { version = "1.19.0" }
Expand Down
8 changes: 0 additions & 8 deletions PIP_COMPATIBILITY.md
Original file line number Diff line number Diff line change
Expand Up @@ -259,14 +259,6 @@ When uv resolutions differ from `pip` in undesirable ways, it's often a sign tha
are too loose, and that the user should consider tightening them. For example, in the case of
`starlette` and `fastapi`, the user could require `fastapi>=0.110.0`.

## Hash-checking mode

While uv will include hashes via `uv pip compile --generate-hashes`, it does not support
hash-checking mode, which is a feature of `pip` that allows users to verify the integrity of
downloaded packages by checking their hashes against those provided in the `requirements.txt` file.

In the future, uv will support hash-checking mode. For more, see [#474](https://github.com/astral-sh/uv/issues/474).

## `pip check`

At present, `uv pip check` will surface the following diagnostics:
Expand Down
4 changes: 0 additions & 4 deletions constraints.txt

This file was deleted.

48 changes: 40 additions & 8 deletions crates/distribution-types/src/cached.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@ use anyhow::Result;

use distribution_filename::WheelFilename;
use pep508_rs::VerbatimUrl;
use pypi_types::HashDigest;
use uv_normalize::PackageName;

use crate::direct_url::{DirectUrl, LocalFileUrl};
use crate::hashed::Hashed;
use crate::{
BuiltDist, Dist, DistributionMetadata, InstalledMetadata, InstalledVersion, Name, SourceDist,
VersionOrUrl,
Expand All @@ -25,6 +27,7 @@ pub enum CachedDist {
pub struct CachedRegistryDist {
pub filename: WheelFilename,
pub path: PathBuf,
pub hashes: Vec<HashDigest>,
}

#[derive(Debug, Clone)]
Expand All @@ -33,45 +36,60 @@ pub struct CachedDirectUrlDist {
pub url: VerbatimUrl,
pub path: PathBuf,
pub editable: bool,
pub hashes: Vec<HashDigest>,
}

impl CachedDist {
/// Initialize a [`CachedDist`] from a [`Dist`].
pub fn from_remote(remote: Dist, filename: WheelFilename, path: PathBuf) -> Self {
pub fn from_remote(
remote: Dist,
filename: WheelFilename,
hashes: Vec<HashDigest>,
path: PathBuf,
) -> Self {
match remote {
Dist::Built(BuiltDist::Registry(_dist)) => {
Self::Registry(CachedRegistryDist { filename, path })
}
Dist::Built(BuiltDist::Registry(_dist)) => Self::Registry(CachedRegistryDist {
filename,
path,
hashes,
}),
Dist::Built(BuiltDist::DirectUrl(dist)) => Self::Url(CachedDirectUrlDist {
filename,
url: dist.url,
hashes,
path,
editable: false,
}),
Dist::Built(BuiltDist::Path(dist)) => Self::Url(CachedDirectUrlDist {
filename,
url: dist.url,
hashes,
path,
editable: false,
}),
Dist::Source(SourceDist::Registry(_dist)) => {
Self::Registry(CachedRegistryDist { filename, path })
}
Dist::Source(SourceDist::Registry(_dist)) => Self::Registry(CachedRegistryDist {
filename,
path,
hashes,
}),
Dist::Source(SourceDist::DirectUrl(dist)) => Self::Url(CachedDirectUrlDist {
filename,
url: dist.url,
hashes,
path,
editable: false,
}),
Dist::Source(SourceDist::Git(dist)) => Self::Url(CachedDirectUrlDist {
filename,
url: dist.url,
hashes,
path,
editable: false,
}),
Dist::Source(SourceDist::Path(dist)) => Self::Url(CachedDirectUrlDist {
filename,
url: dist.url,
hashes,
path,
editable: dist.editable,
}),
Expand Down Expand Up @@ -104,13 +122,15 @@ impl CachedDist {
}
}

/// Returns `true` if the distribution is editable.
pub fn editable(&self) -> bool {
match self {
Self::Registry(_) => false,
Self::Url(dist) => dist.editable,
}
}

/// Returns the [`WheelFilename`] of the distribution.
pub fn filename(&self) -> &WheelFilename {
match self {
Self::Registry(dist) => &dist.filename,
Expand All @@ -119,12 +139,24 @@ impl CachedDist {
}
}

impl Hashed for CachedRegistryDist {
fn hashes(&self) -> &[HashDigest] {
&self.hashes
}
}

impl CachedDirectUrlDist {
/// Initialize a [`CachedDirectUrlDist`] from a [`WheelFilename`], [`url::Url`], and [`Path`].
pub fn from_url(filename: WheelFilename, url: VerbatimUrl, path: PathBuf) -> Self {
pub fn from_url(
filename: WheelFilename,
url: VerbatimUrl,
hashes: Vec<HashDigest>,
path: PathBuf,
) -> Self {
Self {
filename,
url,
hashes,
path,
editable: false,
}
Expand Down
27 changes: 27 additions & 0 deletions crates/distribution-types/src/hashed.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
use pypi_types::HashDigest;

pub trait Hashed {
/// Return the [`HashDigest`]s for the archive.
fn hashes(&self) -> &[HashDigest];

/// Returns `true` if the archive satisfies the given hashes.
fn satisfies(&self, hashes: &[HashDigest]) -> bool {
if hashes.is_empty() {
true
} else {
self.hashes().iter().any(|hash| hashes.contains(hash))
}
}

/// Returns `true` if the archive includes a hash for at least one of the given algorithms.
fn has_digests(&self, hashes: &[HashDigest]) -> bool {
if hashes.is_empty() {
true
} else {
hashes
.iter()
.map(HashDigest::algorithm)
.any(|algorithm| self.hashes().iter().any(|hash| hash.algorithm == algorithm))
}
}
}
2 changes: 2 additions & 0 deletions crates/distribution-types/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ pub use crate::direct_url::*;
pub use crate::editable::*;
pub use crate::error::*;
pub use crate::file::*;
pub use crate::hashed::*;
pub use crate::id::*;
pub use crate::index_url::*;
pub use crate::installed::*;
Expand All @@ -66,6 +67,7 @@ mod direct_url;
mod editable;
mod error;
mod file;
mod hashed;
mod id;
mod index_url;
mod installed;
Expand Down
8 changes: 0 additions & 8 deletions crates/pypi-types/src/simple_json.rs
Original file line number Diff line number Diff line change
Expand Up @@ -159,14 +159,6 @@ impl Hashes {
}
digests
}

/// Returns `true` if the hash is empty.
pub fn is_empty(&self) -> bool {
self.sha512.is_none()
&& self.sha384.is_none()
&& self.sha256.is_none()
&& self.md5.is_none()
}
}

impl FromStr for Hashes {
Expand Down
4 changes: 2 additions & 2 deletions crates/uv-cache/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -594,12 +594,12 @@ pub enum CacheBucket {
impl CacheBucket {
fn to_str(self) -> &'static str {
match self {
Self::BuiltWheels => "built-wheels-v2",
Self::BuiltWheels => "built-wheels-v3",
Self::FlatIndex => "flat-index-v0",
Self::Git => "git-v0",
Self::Interpreter => "interpreter-v0",
Self::Simple => "simple-v7",
Self::Wheels => "wheels-v0",
Self::Wheels => "wheels-v1",
Self::Archive => "archive-v0",
}
}
Expand Down
29 changes: 29 additions & 0 deletions crates/uv-client/src/cached_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,35 @@ impl CachedClient {
}
}

/// Make a request without checking whether the cache is fresh.
#[instrument(skip_all)]
pub async fn skip_cache<
Payload: Serialize + DeserializeOwned + Send + 'static,
CallBackError,
Callback,
CallbackReturn,
>(
&self,
req: Request,
cache_entry: &CacheEntry,
response_callback: Callback,
) -> Result<Payload, CachedClientError<CallBackError>>
where
Callback: FnOnce(Response) -> CallbackReturn + Send,
CallbackReturn: Future<Output = Result<Payload, CallBackError>> + Send,
{
let (response, cache_policy) = self.fresh_request(req).await?;

let payload = self
.run_response_callback(cache_entry, cache_policy, response, move |resp| async {
let payload = response_callback(resp).await?;
Ok(SerdeCacheable { inner: payload })
})
.await?;

Ok(payload)
}

async fn resend_and_heal_cache<Payload: Cacheable, CallBackError, Callback, CallbackReturn>(
&self,
req: Request,
Expand Down
5 changes: 3 additions & 2 deletions crates/uv-client/src/flat_index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use std::path::PathBuf;
use futures::{FutureExt, StreamExt};
use reqwest::Response;
use rustc_hash::FxHashMap;
use tracing::{debug, info_span, instrument, Instrument, warn};
use tracing::{debug, info_span, instrument, warn, Instrument};
use url::Url;

use distribution_filename::{DistFilename, SourceDistFilename, WheelFilename};
Expand All @@ -23,9 +23,9 @@ use uv_configuration::{NoBinary, NoBuild};
use uv_normalize::PackageName;
use uv_types::RequiredHashes;

use crate::{Connectivity, Error, ErrorKind, RegistryClient};
use crate::cached_client::{CacheControl, CachedClientError};
use crate::html::SimpleHtml;
use crate::{Connectivity, Error, ErrorKind, RegistryClient};

#[derive(Debug, thiserror::Error)]
pub enum FlatIndexError {
Expand Down Expand Up @@ -303,6 +303,7 @@ impl FlatIndex {
Self { index, offline }
}

#[allow(clippy::too_many_arguments)]
fn add_file(
distributions: &mut FlatDistributions,
file: File,
Expand Down
4 changes: 3 additions & 1 deletion crates/uv-dev/src/resolve_cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> {
let index_locations =
IndexLocations::new(args.index_url, args.extra_index_url, args.find_links, false);
let index = InMemoryIndex::default();
let hashes = RequiredHashes::default();
let in_flight = InFlight::default();
let no_build = if args.no_build {
NoBuild::All
Expand Down Expand Up @@ -100,14 +101,15 @@ pub(crate) async fn resolve_cli(args: ResolveCliArgs) -> Result<()> {
// Copied from `BuildDispatch`
let tags = venv.interpreter().tags()?;
let resolver = Resolver::new(
Manifest::simple(args.requirements.clone(), RequiredHashes::default()),
Manifest::simple(args.requirements.clone()),
Options::default(),
venv.interpreter().markers(),
venv.interpreter(),
tags,
&client,
&flat_index,
&index,
&hashes,
&build_dispatch,
&site_packages,
)?;
Expand Down
Loading

0 comments on commit b1e9f80

Please sign in to comment.