From bb61513952d54f239c74a83adbc217800d902934 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Thu, 5 Sep 2024 14:30:10 -0400 Subject: [PATCH] Respect hashes in constraints files (#7093) ## Summary Like pip, if hashes are present on both the requirement and the constraint, we prefer the requirement. Closes #7089. --- crates/bench/benches/uv.rs | 7 +- .../src/specified_requirement.rs | 38 +++++-- crates/pypi-types/src/requirement.rs | 13 ++- crates/uv-dispatch/src/lib.rs | 13 ++- crates/uv-installer/src/site_packages.rs | 11 +- crates/uv-requirements/src/specification.rs | 13 ++- crates/uv-types/src/hash.rs | 82 ++++++++++++-- crates/uv/src/commands/build.rs | 8 +- crates/uv/src/commands/pip/compile.rs | 25 ++++- crates/uv/src/commands/pip/install.rs | 40 ++++++- crates/uv/src/commands/pip/operations.rs | 11 +- crates/uv/src/commands/pip/sync.rs | 28 ++++- crates/uv/src/commands/project/add.rs | 14 ++- crates/uv/src/commands/project/lock.rs | 18 ++- crates/uv/src/commands/project/mod.rs | 26 +++-- crates/uv/src/commands/project/sync.rs | 10 +- crates/uv/src/commands/venv.rs | 10 +- crates/uv/tests/pip_install.rs | 105 ++++++++++++++++-- crates/uv/tests/pip_sync.rs | 14 +-- 19 files changed, 383 insertions(+), 103 deletions(-) diff --git a/crates/bench/benches/uv.rs b/crates/bench/benches/uv.rs index 2076c03d809e..badbb089deda 100644 --- a/crates/bench/benches/uv.rs +++ b/crates/bench/benches/uv.rs @@ -92,7 +92,7 @@ mod resolver { use uv_cache::Cache; use uv_client::RegistryClient; use uv_configuration::{ - BuildOptions, Concurrency, ConfigSettings, IndexStrategy, SourceStrategy, + BuildOptions, Concurrency, ConfigSettings, Constraints, IndexStrategy, SourceStrategy, }; use uv_dispatch::BuildDispatch; use uv_distribution::DistributionDatabase; @@ -159,7 +159,7 @@ mod resolver { let installed_packages = EmptyInstalledPackages; let sources = SourceStrategy::default(); let options = OptionsBuilder::new().exclude_newer(exclude_newer).build(); - let build_constraints = []; + let build_constraints = Constraints::default(); let python_requirement = if universal { PythonRequirement::from_requires_python( @@ -173,7 +173,7 @@ mod resolver { let build_context = BuildDispatch::new( client, &cache, - &build_constraints, + build_constraints, interpreter, &index_locations, &flat_index, @@ -185,6 +185,7 @@ mod resolver { build_isolation, LinkMode::default(), &build_options, + &hashes, exclude_newer, sources, concurrency, diff --git a/crates/distribution-types/src/specified_requirement.rs b/crates/distribution-types/src/specified_requirement.rs index 6ebb431892c8..f9bc2fc2d008 100644 --- a/crates/distribution-types/src/specified_requirement.rs +++ b/crates/distribution-types/src/specified_requirement.rs @@ -7,6 +7,16 @@ use uv_normalize::ExtraName; use crate::VerbatimParsedUrl; +/// An [`UnresolvedRequirement`] with additional metadata from `requirements.txt`, currently only +/// hashes but in the future also editable and similar information. +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct NameRequirementSpecification { + /// The actual requirement. + pub requirement: Requirement, + /// Hashes of the downloadable packages. + pub hashes: Vec, +} + /// An [`UnresolvedRequirement`] with additional metadata from `requirements.txt`, currently only /// hashes but in the future also editable and similar information. #[derive(Debug, Clone, Eq, PartialEq, Hash)] @@ -86,13 +96,7 @@ impl UnresolvedRequirement { /// Return the hashes of the requirement, as specified in the URL fragment. pub fn hashes(&self) -> Option { match self { - Self::Named(requirement) => { - let RequirementSource::Url { ref url, .. } = requirement.source else { - return None; - }; - let fragment = url.fragment()?; - Hashes::parse_fragment(fragment).ok() - } + Self::Named(requirement) => requirement.hashes(), Self::Unnamed(requirement) => { let ParsedUrl::Archive(ref url) = requirement.url.parsed_url else { return None; @@ -104,6 +108,17 @@ impl UnresolvedRequirement { } } +impl NameRequirementSpecification { + /// Return the hashes of the requirement, as specified in the URL fragment. + pub fn hashes(&self) -> Option { + let RequirementSource::Url { ref url, .. } = self.requirement.source else { + return None; + }; + let fragment = url.fragment()?; + Hashes::parse_fragment(fragment).ok() + } +} + impl From for UnresolvedRequirementSpecification { fn from(requirement: Requirement) -> Self { Self { @@ -112,3 +127,12 @@ impl From for UnresolvedRequirementSpecification { } } } + +impl From for NameRequirementSpecification { + fn from(requirement: Requirement) -> Self { + Self { + requirement, + hashes: Vec::new(), + } + } +} diff --git a/crates/pypi-types/src/requirement.rs b/crates/pypi-types/src/requirement.rs index babc8501bc6c..2d1854e2fbe8 100644 --- a/crates/pypi-types/src/requirement.rs +++ b/crates/pypi-types/src/requirement.rs @@ -16,8 +16,8 @@ use uv_git::{GitReference, GitSha, GitUrl}; use uv_normalize::{ExtraName, PackageName}; use crate::{ - ParsedArchiveUrl, ParsedDirectoryUrl, ParsedGitUrl, ParsedPathUrl, ParsedUrl, ParsedUrlError, - VerbatimParsedUrl, + Hashes, ParsedArchiveUrl, ParsedDirectoryUrl, ParsedGitUrl, ParsedPathUrl, ParsedUrl, + ParsedUrlError, VerbatimParsedUrl, }; #[derive(Debug, Error)] @@ -114,6 +114,15 @@ impl Requirement { ..self }) } + + /// Return the hashes of the requirement, as specified in the URL fragment. + pub fn hashes(&self) -> Option { + let RequirementSource::Url { ref url, .. } = self.source else { + return None; + }; + let fragment = url.fragment()?; + Hashes::parse_fragment(fragment).ok() + } } impl From for pep508_rs::Requirement { diff --git a/crates/uv-dispatch/src/lib.rs b/crates/uv-dispatch/src/lib.rs index ea23d86facb2..d3643137d7c1 100644 --- a/crates/uv-dispatch/src/lib.rs +++ b/crates/uv-dispatch/src/lib.rs @@ -47,6 +47,7 @@ pub struct BuildDispatch<'a> { link_mode: install_wheel_rs::linker::LinkMode, build_options: &'a BuildOptions, config_settings: &'a ConfigSettings, + hasher: &'a HashStrategy, exclude_newer: Option, source_build_context: SourceBuildContext, build_extra_env_vars: FxHashMap, @@ -58,7 +59,7 @@ impl<'a> BuildDispatch<'a> { pub fn new( client: &'a RegistryClient, cache: &'a Cache, - constraints: &'a [Requirement], + constraints: Constraints, interpreter: &'a Interpreter, index_locations: &'a IndexLocations, flat_index: &'a FlatIndex, @@ -70,6 +71,7 @@ impl<'a> BuildDispatch<'a> { build_isolation: BuildIsolation<'a>, link_mode: install_wheel_rs::linker::LinkMode, build_options: &'a BuildOptions, + hasher: &'a HashStrategy, exclude_newer: Option, sources: SourceStrategy, concurrency: Concurrency, @@ -77,7 +79,7 @@ impl<'a> BuildDispatch<'a> { Self { client, cache, - constraints: Constraints::from_requirements(constraints.iter().cloned()), + constraints, interpreter, index_locations, flat_index, @@ -89,6 +91,7 @@ impl<'a> BuildDispatch<'a> { build_isolation, link_mode, build_options, + hasher, exclude_newer, source_build_context: SourceBuildContext::default(), build_extra_env_vars: FxHashMap::default(), @@ -152,7 +155,7 @@ impl<'a> BuildContext for BuildDispatch<'a> { Some(tags), self.flat_index, self.index, - &HashStrategy::None, + self.hasher, self, EmptyInstalledPackages, DistributionDatabase::new(self.client, self, self.concurrency.downloads), @@ -205,7 +208,7 @@ impl<'a> BuildContext for BuildDispatch<'a> { site_packages, &Reinstall::default(), &BuildOptions::default(), - &HashStrategy::default(), + self.hasher, self.index_locations, self.cache(), venv, @@ -238,7 +241,7 @@ impl<'a> BuildContext for BuildDispatch<'a> { let preparer = Preparer::new( self.cache, tags, - &HashStrategy::None, + self.hasher, self.build_options, DistributionDatabase::new(self.client, self, self.concurrency.downloads), ); diff --git a/crates/uv-installer/src/site_packages.rs b/crates/uv-installer/src/site_packages.rs index b6de14853155..56939a288303 100644 --- a/crates/uv-installer/src/site_packages.rs +++ b/crates/uv-installer/src/site_packages.rs @@ -8,7 +8,8 @@ use rustc_hash::{FxBuildHasher, FxHashMap, FxHashSet}; use url::Url; use distribution_types::{ - Diagnostic, InstalledDist, Name, UnresolvedRequirement, UnresolvedRequirementSpecification, + Diagnostic, InstalledDist, Name, NameRequirementSpecification, UnresolvedRequirement, + UnresolvedRequirementSpecification, }; use pep440_rs::{Version, VersionSpecifiers}; use pypi_types::{Requirement, ResolverMarkerEnvironment, VerbatimParsedUrl}; @@ -283,18 +284,18 @@ impl SitePackages { pub fn satisfies( &self, requirements: &[UnresolvedRequirementSpecification], - constraints: &[Requirement], + constraints: &[NameRequirementSpecification], markers: &ResolverMarkerEnvironment, ) -> Result { // Collect the constraints. let constraints: FxHashMap<&PackageName, Vec<&Requirement>> = constraints .iter() - .fold(FxHashMap::default(), |mut constraints, requirement| { + .fold(FxHashMap::default(), |mut constraints, constraint| { constraints - .entry(&requirement.name) + .entry(&constraint.requirement.name) .or_default() - .push(requirement); + .push(&constraint.requirement); constraints }); diff --git a/crates/uv-requirements/src/specification.rs b/crates/uv-requirements/src/specification.rs index 211423b036f9..d1943e5e1ab1 100644 --- a/crates/uv-requirements/src/specification.rs +++ b/crates/uv-requirements/src/specification.rs @@ -35,7 +35,8 @@ use tracing::instrument; use cache_key::CanonicalUrl; use distribution_types::{ - FlatIndexLocation, IndexUrl, UnresolvedRequirement, UnresolvedRequirementSpecification, + FlatIndexLocation, IndexUrl, NameRequirementSpecification, UnresolvedRequirement, + UnresolvedRequirementSpecification, }; use pep508_rs::{MarkerTree, UnnamedRequirement, UnnamedRequirementUrl}; use pypi_types::Requirement; @@ -56,7 +57,7 @@ pub struct RequirementsSpecification { /// The requirements for the project. pub requirements: Vec, /// The constraints for the project. - pub constraints: Vec, + pub constraints: Vec, /// The overrides for the project. pub overrides: Vec, /// The source trees from which to extract requirements. @@ -129,6 +130,7 @@ impl RequirementsSpecification { .constraints .into_iter() .map(Requirement::from) + .map(NameRequirementSpecification::from) .collect(), index_url: requirements_txt.index_url.map(IndexUrl::from), extra_index_urls: requirements_txt @@ -247,13 +249,16 @@ impl RequirementsSpecification { } // Read all constraints, treating both requirements _and_ constraints as constraints. - // Overrides are ignored, as are the hashes, as they are not relevant for constraints. + // Overrides are ignored. for source in constraints { let source = Self::from_source(source, client_builder).await?; for entry in source.requirements { match entry.requirement { UnresolvedRequirement::Named(requirement) => { - spec.constraints.push(requirement); + spec.constraints.push(NameRequirementSpecification { + requirement, + hashes: entry.hashes, + }); } UnresolvedRequirement::Unnamed(requirement) => { return Err(anyhow::anyhow!( diff --git a/crates/uv-types/src/hash.rs b/crates/uv-types/src/hash.rs index 54477de8a59f..0b512650c963 100644 --- a/crates/uv-types/src/hash.rs +++ b/crates/uv-types/src/hash.rs @@ -126,13 +126,56 @@ impl HashStrategy { /// to "only evaluate marker expressions that reference an extra name.") pub fn from_requirements<'a>( requirements: impl Iterator, + constraints: impl Iterator, marker_env: Option<&ResolverMarkerEnvironment>, mode: HashCheckingMode, ) -> Result { - let mut hashes = FxHashMap::>::default(); + let mut constraint_hashes = FxHashMap::>::default(); + + // First, index the constraints by name. + for (requirement, digests) in constraints { + if !requirement + .evaluate_markers(marker_env.map(ResolverMarkerEnvironment::markers), &[]) + { + continue; + } + + // Every constraint must be a pinned version. + let Some(id) = Self::pin(requirement) else { + if mode.is_require() { + return Err(HashStrategyError::UnpinnedRequirement( + requirement.to_string(), + mode, + )); + } + continue; + }; + + let digests = if digests.is_empty() { + // If there are no hashes, and the distribution is URL-based, attempt to extract + // it from the fragment. + requirement + .hashes() + .map(Hashes::into_digests) + .unwrap_or_default() + } else { + // Parse the hashes. + digests + .iter() + .map(|digest| HashDigest::from_str(digest)) + .collect::, _>>()? + }; + + if digests.is_empty() { + continue; + } + + constraint_hashes.insert(id, digests); + } // For each requirement, map from name to allowed hashes. We use the last entry for each // package. + let mut requirement_hashes = FxHashMap::>::default(); for (requirement, digests) in requirements { if !requirement .evaluate_markers(marker_env.map(ResolverMarkerEnvironment::markers), &[]) @@ -143,9 +186,17 @@ impl HashStrategy { // Every requirement must be either a pinned version or a direct URL. let id = match &requirement { UnresolvedRequirement::Named(requirement) => { - Self::pin(requirement).ok_or_else(|| { - HashStrategyError::UnpinnedRequirement(requirement.to_string(), mode) - })? + if let Some(id) = Self::pin(requirement) { + id + } else { + if mode.is_require() { + return Err(HashStrategyError::UnpinnedRequirement( + requirement.to_string(), + mode, + )); + } + continue; + } } UnresolvedRequirement::Unnamed(requirement) => { // Direct URLs are always allowed. @@ -168,20 +219,27 @@ impl HashStrategy { .collect::, _>>()? }; + // Under `--require-hashes`, every requirement must include a hash. if digests.is_empty() { - // Under `--require-hashes`, every requirement must include a hash. if mode.is_require() { - return Err(HashStrategyError::MissingHashes( - requirement.to_string(), - mode, - )); + if constraint_hashes.get(&id).map_or(true, Vec::is_empty) { + return Err(HashStrategyError::MissingHashes( + requirement.to_string(), + mode, + )); + } } continue; } - hashes.insert(id, digests); + requirement_hashes.insert(id, digests); } + // Merge the hashes, preferring requirements over constraints, to match pip. + let hashes: FxHashMap> = constraint_hashes + .into_iter() + .chain(requirement_hashes) + .collect(); match mode { HashCheckingMode::Verify => Ok(Self::Verify(Arc::new(hashes))), HashCheckingMode::Require => Ok(Self::Require(Arc::new(hashes))), @@ -248,9 +306,9 @@ pub enum HashStrategyError { #[error(transparent)] Hash(#[from] HashError), #[error( - "In `{1}` mode, all requirement must have their versions pinned with `==`, but found: {0}" + "In `{1}` mode, all requirements must have their versions pinned with `==`, but found: {0}" )] UnpinnedRequirement(String, HashCheckingMode), - #[error("In `{1}` mode, all requirement must have a hash, but none were provided for: {0}")] + #[error("In `{1}` mode, all requirements must have a hash, but none were provided for: {0}")] MissingHashes(String, HashCheckingMode), } diff --git a/crates/uv/src/commands/build.rs b/crates/uv/src/commands/build.rs index e8ed68dabaa8..67f54676cb1e 100644 --- a/crates/uv/src/commands/build.rs +++ b/crates/uv/src/commands/build.rs @@ -12,7 +12,7 @@ use std::path::{Path, PathBuf}; use uv_auth::store_credentials_from_url; use uv_cache::Cache; use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder}; -use uv_configuration::{BuildKind, BuildOutput, Concurrency}; +use uv_configuration::{BuildKind, BuildOutput, Concurrency, Constraints}; use uv_dispatch::BuildDispatch; use uv_fs::{Simplified, CWD}; use uv_normalize::PackageName; @@ -251,7 +251,8 @@ async fn build_impl( // TODO(charlie): These are all default values. We should consider whether we want to make them // optional on the downstream APIs. - let build_constraints = []; + let build_constraints = Constraints::default(); + let build_hasher = HashStrategy::default(); let hasher = HashStrategy::None; // Resolve the flat indexes from `--find-links`. @@ -268,7 +269,7 @@ async fn build_impl( let build_dispatch = BuildDispatch::new( &client, cache, - &build_constraints, + build_constraints, &interpreter, index_locations, &flat_index, @@ -280,6 +281,7 @@ async fn build_impl( build_isolation, link_mode, build_options, + &build_hasher, exclude_newer, sources, concurrency, diff --git a/crates/uv/src/commands/pip/compile.rs b/crates/uv/src/commands/pip/compile.rs index cab2f9a20c8d..d65c424aa592 100644 --- a/crates/uv/src/commands/pip/compile.rs +++ b/crates/uv/src/commands/pip/compile.rs @@ -9,15 +9,17 @@ use itertools::Itertools; use owo_colors::OwoColorize; use tracing::debug; -use distribution_types::{IndexLocations, UnresolvedRequirementSpecification, Verbatim}; +use distribution_types::{ + IndexLocations, NameRequirementSpecification, UnresolvedRequirementSpecification, Verbatim, +}; use install_wheel_rs::linker::LinkMode; use pypi_types::{Requirement, SupportedEnvironments}; use uv_auth::store_credentials_from_url; use uv_cache::Cache; use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder}; use uv_configuration::{ - BuildOptions, Concurrency, ConfigSettings, ExtrasSpecification, IndexStrategy, NoBinary, - NoBuild, Reinstall, SourceStrategy, TrustedHost, Upgrade, + BuildOptions, Concurrency, ConfigSettings, Constraints, ExtrasSpecification, IndexStrategy, + NoBinary, NoBuild, Reinstall, SourceStrategy, TrustedHost, Upgrade, }; use uv_configuration::{KeyringProviderType, TargetTriple}; use uv_dispatch::BuildDispatch; @@ -136,7 +138,11 @@ pub(crate) async fn pip_compile( let constraints = constraints .iter() .cloned() - .chain(constraints_from_workspace.into_iter()) + .chain( + constraints_from_workspace + .into_iter() + .map(NameRequirementSpecification::from), + ) .collect(); let overrides: Vec = overrides @@ -314,10 +320,18 @@ pub(crate) async fn pip_compile( BuildIsolation::SharedPackage(&environment, &no_build_isolation_package) }; + // Don't enforce hashes in `pip compile`. + let build_constraints = Constraints::from_requirements( + build_constraints + .iter() + .map(|constraint| constraint.requirement.clone()), + ); + let build_hashes = HashStrategy::None; + let build_dispatch = BuildDispatch::new( &client, &cache, - &build_constraints, + build_constraints, &interpreter, &index_locations, &flat_index, @@ -329,6 +343,7 @@ pub(crate) async fn pip_compile( build_isolation, link_mode, &build_options, + &build_hashes, exclude_newer, sources, concurrency, diff --git a/crates/uv/src/commands/pip/install.rs b/crates/uv/src/commands/pip/install.rs index 0dae3aa7410c..7223c671e0d2 100644 --- a/crates/uv/src/commands/pip/install.rs +++ b/crates/uv/src/commands/pip/install.rs @@ -5,7 +5,9 @@ use itertools::Itertools; use owo_colors::OwoColorize; use tracing::{debug, enabled, Level}; -use distribution_types::{IndexLocations, Resolution, UnresolvedRequirementSpecification}; +use distribution_types::{ + IndexLocations, NameRequirementSpecification, Resolution, UnresolvedRequirementSpecification, +}; use install_wheel_rs::linker::LinkMode; use pep508_rs::PackageName; use pypi_types::Requirement; @@ -13,7 +15,7 @@ use uv_auth::store_credentials_from_url; use uv_cache::Cache; use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder}; use uv_configuration::{ - BuildOptions, Concurrency, ConfigSettings, ExtrasSpecification, HashCheckingMode, + BuildOptions, Concurrency, ConfigSettings, Constraints, ExtrasSpecification, HashCheckingMode, IndexStrategy, Reinstall, SourceStrategy, TrustedHost, Upgrade, }; use uv_configuration::{KeyringProviderType, TargetTriple}; @@ -114,10 +116,14 @@ pub(crate) async fn pip_install( let build_constraints = operations::read_constraints(build_constraints, &client_builder).await?; - let constraints: Vec = constraints + let constraints: Vec = constraints .iter() .cloned() - .chain(constraints_from_workspace.into_iter()) + .chain( + constraints_from_workspace + .into_iter() + .map(NameRequirementSpecification::from), + ) .collect(); let overrides: Vec = overrides @@ -247,6 +253,9 @@ pub(crate) async fn pip_install( .iter() .chain(overrides.iter()) .map(|entry| (&entry.requirement, entry.hashes.as_slice())), + constraints + .iter() + .map(|entry| (&entry.requirement, entry.hashes.as_slice())), Some(&markers), hash_checking, )? @@ -297,6 +306,26 @@ pub(crate) async fn pip_install( BuildIsolation::SharedPackage(&environment, &no_build_isolation_package) }; + // Enforce (but never require) the build constraints, if `--require-hashes` or `--verify-hashes` + // is provided. _Requiring_ hashes would be too strict, and would break with pip. + let build_hasher = if hash_checking.is_some() { + HashStrategy::from_requirements( + std::iter::empty(), + build_constraints + .iter() + .map(|entry| (&entry.requirement, entry.hashes.as_slice())), + Some(&markers), + HashCheckingMode::Verify, + )? + } else { + HashStrategy::None + }; + let build_constraints = Constraints::from_requirements( + build_constraints + .iter() + .map(|constraint| constraint.requirement.clone()), + ); + // Initialize any shared state. let state = SharedState::default(); @@ -304,7 +333,7 @@ pub(crate) async fn pip_install( let build_dispatch = BuildDispatch::new( &client, &cache, - &build_constraints, + build_constraints, interpreter, &index_locations, &flat_index, @@ -316,6 +345,7 @@ pub(crate) async fn pip_install( build_isolation, link_mode, &build_options, + &build_hasher, exclude_newer, sources, concurrency, diff --git a/crates/uv/src/commands/pip/operations.rs b/crates/uv/src/commands/pip/operations.rs index 11bf42626c80..9801290c4215 100644 --- a/crates/uv/src/commands/pip/operations.rs +++ b/crates/uv/src/commands/pip/operations.rs @@ -9,15 +9,15 @@ use std::path::PathBuf; use tracing::debug; use distribution_types::{ - CachedDist, Diagnostic, InstalledDist, LocalDist, ResolutionDiagnostic, - UnresolvedRequirementSpecification, + CachedDist, Diagnostic, InstalledDist, LocalDist, NameRequirementSpecification, + ResolutionDiagnostic, UnresolvedRequirementSpecification, }; use distribution_types::{ DistributionMetadata, IndexLocations, InstalledMetadata, Name, Resolution, }; use install_wheel_rs::linker::LinkMode; use platform_tags::Tags; -use pypi_types::{Requirement, ResolverMarkerEnvironment}; +use pypi_types::ResolverMarkerEnvironment; use uv_cache::Cache; use uv_client::{BaseClientBuilder, RegistryClient}; use uv_configuration::{ @@ -76,7 +76,7 @@ pub(crate) async fn read_requirements( pub(crate) async fn read_constraints( constraints: &[RequirementsSource], client_builder: &BaseClientBuilder<'_>, -) -> Result, Error> { +) -> Result, Error> { Ok( RequirementsSpecification::from_sources(&[], constraints, &[], client_builder) .await? @@ -87,7 +87,7 @@ pub(crate) async fn read_constraints( /// Resolve a set of requirements, similar to running `pip compile`. pub(crate) async fn resolve( requirements: Vec, - constraints: Vec, + constraints: Vec, overrides: Vec, dev: Vec, source_trees: Vec, @@ -196,6 +196,7 @@ pub(crate) async fn resolve( let constraints = Constraints::from_requirements( constraints .into_iter() + .map(|constraint| constraint.requirement) .chain(upgrade.constraints().cloned()), ); let overrides = Overrides::from_requirements(overrides); diff --git a/crates/uv/src/commands/pip/sync.rs b/crates/uv/src/commands/pip/sync.rs index e17242b54283..383f5a880a87 100644 --- a/crates/uv/src/commands/pip/sync.rs +++ b/crates/uv/src/commands/pip/sync.rs @@ -12,7 +12,7 @@ use uv_auth::store_credentials_from_url; use uv_cache::Cache; use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder}; use uv_configuration::{ - BuildOptions, Concurrency, ConfigSettings, ExtrasSpecification, HashCheckingMode, + BuildOptions, Concurrency, ConfigSettings, Constraints, ExtrasSpecification, HashCheckingMode, IndexStrategy, Reinstall, SourceStrategy, TrustedHost, Upgrade, }; use uv_configuration::{KeyringProviderType, TargetTriple}; @@ -203,6 +203,9 @@ pub(crate) async fn pip_sync( requirements .iter() .map(|entry| (&entry.requirement, entry.hashes.as_slice())), + constraints + .iter() + .map(|entry| (&entry.requirement, entry.hashes.as_slice())), Some(&markers), hash_checking, )? @@ -247,6 +250,26 @@ pub(crate) async fn pip_sync( BuildIsolation::SharedPackage(&environment, &no_build_isolation_package) }; + // Enforce (but never require) the build constraints, if `--require-hashes` or `--verify-hashes` + // is provided. _Requiring_ hashes would be too strict, and would break with pip. + let build_hasher = if hash_checking.is_some() { + HashStrategy::from_requirements( + std::iter::empty(), + build_constraints + .iter() + .map(|entry| (&entry.requirement, entry.hashes.as_slice())), + Some(&markers), + HashCheckingMode::Verify, + )? + } else { + HashStrategy::None + }; + let build_constraints = Constraints::from_requirements( + build_constraints + .iter() + .map(|constraint| constraint.requirement.clone()), + ); + // Initialize any shared state. let state = SharedState::default(); @@ -260,7 +283,7 @@ pub(crate) async fn pip_sync( let build_dispatch = BuildDispatch::new( &client, &cache, - &build_constraints, + build_constraints, interpreter, &index_locations, &flat_index, @@ -272,6 +295,7 @@ pub(crate) async fn pip_sync( build_isolation, link_mode, &build_options, + &build_hasher, exclude_newer, sources, concurrency, diff --git a/crates/uv/src/commands/project/add.rs b/crates/uv/src/commands/project/add.rs index 8bb353078a1c..7546af78125e 100644 --- a/crates/uv/src/commands/project/add.rs +++ b/crates/uv/src/commands/project/add.rs @@ -13,7 +13,9 @@ use pypi_types::redact_git_credentials; use uv_auth::{store_credentials_from_url, Credentials}; use uv_cache::Cache; use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder}; -use uv_configuration::{Concurrency, ExtrasSpecification, InstallOptions, SourceStrategy}; +use uv_configuration::{ + Concurrency, Constraints, ExtrasSpecification, InstallOptions, SourceStrategy, +}; use uv_dispatch::BuildDispatch; use uv_distribution::DistributionDatabase; use uv_fs::{Simplified, CWD}; @@ -241,10 +243,11 @@ pub(crate) async fn add( // TODO(charlie): These are all default values. We should consider whether we want to make them // optional on the downstream APIs. - let python_version = None; - let python_platform = None; + let build_constraints = Constraints::default(); + let build_hasher = HashStrategy::default(); let hasher = HashStrategy::default(); - let build_constraints = []; + let python_platform = None; + let python_version = None; let sources = SourceStrategy::Enabled; // Determine the environment for the resolution. @@ -290,7 +293,7 @@ pub(crate) async fn add( let build_dispatch = BuildDispatch::new( &client, cache, - &build_constraints, + build_constraints, target.interpreter(), &settings.index_locations, &flat_index, @@ -302,6 +305,7 @@ pub(crate) async fn add( build_isolation, settings.link_mode, &settings.build_options, + &build_hasher, settings.exclude_newer, sources, concurrency, diff --git a/crates/uv/src/commands/project/lock.rs b/crates/uv/src/commands/project/lock.rs index 7bec6c646dab..d5003a855402 100644 --- a/crates/uv/src/commands/project/lock.rs +++ b/crates/uv/src/commands/project/lock.rs @@ -8,13 +8,15 @@ use owo_colors::OwoColorize; use rustc_hash::{FxBuildHasher, FxHashMap}; use tracing::debug; -use distribution_types::{IndexLocations, UnresolvedRequirementSpecification}; +use distribution_types::{ + IndexLocations, NameRequirementSpecification, UnresolvedRequirementSpecification, +}; use pep440_rs::Version; use pypi_types::{Requirement, SupportedEnvironments}; use uv_auth::store_credentials_from_url; use uv_cache::Cache; use uv_client::{Connectivity, FlatIndexClient, RegistryClientBuilder}; -use uv_configuration::{Concurrency, ExtrasSpecification, Reinstall, Upgrade}; +use uv_configuration::{Concurrency, Constraints, ExtrasSpecification, Reinstall, Upgrade}; use uv_dispatch::BuildDispatch; use uv_distribution::DistributionDatabase; use uv_fs::CWD; @@ -386,7 +388,8 @@ async fn do_lock( // TODO(charlie): These are all default values. We should consider whether we want to make them // optional on the downstream APIs. - let build_constraints = []; + let build_constraints = Constraints::default(); + let build_hasher = HashStrategy::default(); let extras = ExtrasSpecification::default(); // Resolve the flat indexes from `--find-links`. @@ -400,7 +403,7 @@ async fn do_lock( let build_dispatch = BuildDispatch::new( &client, cache, - &build_constraints, + build_constraints, interpreter, index_locations, &flat_index, @@ -412,6 +415,7 @@ async fn do_lock( build_isolation, link_mode, build_options, + &build_hasher, exclude_newer, sources, concurrency, @@ -504,7 +508,11 @@ async fn do_lock( .chain(requirements.iter().cloned()) .map(UnresolvedRequirementSpecification::from) .collect(), - constraints.clone(), + constraints + .iter() + .cloned() + .map(NameRequirementSpecification::from) + .collect(), overrides .iter() .cloned() diff --git a/crates/uv/src/commands/project/mod.rs b/crates/uv/src/commands/project/mod.rs index a10a3b65c7f2..8f5ed2dc5200 100644 --- a/crates/uv/src/commands/project/mod.rs +++ b/crates/uv/src/commands/project/mod.rs @@ -12,7 +12,7 @@ use pypi_types::Requirement; use uv_auth::store_credentials_from_url; use uv_cache::Cache; use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder}; -use uv_configuration::{Concurrency, ExtrasSpecification, Reinstall, Upgrade}; +use uv_configuration::{Concurrency, Constraints, ExtrasSpecification, Reinstall, Upgrade}; use uv_dispatch::BuildDispatch; use uv_distribution::DistributionDatabase; use uv_fs::Simplified; @@ -506,13 +506,14 @@ pub(crate) async fn resolve_names( // optional on the downstream APIs. let hasher = HashStrategy::default(); let flat_index = FlatIndex::default(); - let build_constraints = []; + let build_constraints = Constraints::default(); + let build_hasher = HashStrategy::default(); // Create a build dispatch. let build_dispatch = BuildDispatch::new( &client, cache, - &build_constraints, + build_constraints, interpreter, index_locations, &flat_index, @@ -524,6 +525,7 @@ pub(crate) async fn resolve_names( build_isolation, *link_mode, build_options, + &build_hasher, *exclude_newer, *sources, concurrency, @@ -630,7 +632,8 @@ pub(crate) async fn resolve_environment<'a>( let extras = ExtrasSpecification::default(); let hasher = HashStrategy::default(); let preferences = Vec::default(); - let build_constraints = []; + let build_constraints = Constraints::default(); + let build_hasher = HashStrategy::default(); // When resolving from an interpreter, we assume an empty environment, so reinstalls and // upgrades aren't relevant. @@ -648,7 +651,7 @@ pub(crate) async fn resolve_environment<'a>( let resolve_dispatch = BuildDispatch::new( &client, cache, - &build_constraints, + build_constraints, interpreter, index_locations, &flat_index, @@ -660,6 +663,7 @@ pub(crate) async fn resolve_environment<'a>( build_isolation, link_mode, build_options, + &build_hasher, exclude_newer, sources, concurrency, @@ -759,7 +763,8 @@ pub(crate) async fn sync_environment( // TODO(charlie): These are all default values. We should consider whether we want to make them // optional on the downstream APIs. - let build_constraints = []; + let build_constraints = Constraints::default(); + let build_hasher = HashStrategy::default(); let dry_run = false; let hasher = HashStrategy::default(); @@ -774,7 +779,7 @@ pub(crate) async fn sync_environment( let build_dispatch = BuildDispatch::new( &client, cache, - &build_constraints, + build_constraints, interpreter, index_locations, &flat_index, @@ -786,6 +791,7 @@ pub(crate) async fn sync_environment( build_isolation, link_mode, build_options, + &build_hasher, exclude_newer, sources, concurrency, @@ -949,7 +955,8 @@ pub(crate) async fn update_environment( // TODO(charlie): These are all default values. We should consider whether we want to make them // optional on the downstream APIs. - let build_constraints = []; + let build_constraints = Constraints::default(); + let build_hasher = HashStrategy::default(); let dev = Vec::default(); let dry_run = false; let extras = ExtrasSpecification::default(); @@ -971,7 +978,7 @@ pub(crate) async fn update_environment( let build_dispatch = BuildDispatch::new( &client, cache, - &build_constraints, + build_constraints, interpreter, index_locations, &flat_index, @@ -983,6 +990,7 @@ pub(crate) async fn update_environment( build_isolation, *link_mode, build_options, + &build_hasher, *exclude_newer, *sources, concurrency, diff --git a/crates/uv/src/commands/project/sync.rs b/crates/uv/src/commands/project/sync.rs index eacc608ff8b8..a46797882c0b 100644 --- a/crates/uv/src/commands/project/sync.rs +++ b/crates/uv/src/commands/project/sync.rs @@ -6,7 +6,9 @@ use pep508_rs::MarkerTree; use uv_auth::store_credentials_from_url; use uv_cache::Cache; use uv_client::{Connectivity, FlatIndexClient, RegistryClientBuilder}; -use uv_configuration::{Concurrency, ExtrasSpecification, HashCheckingMode, InstallOptions}; +use uv_configuration::{ + Concurrency, Constraints, ExtrasSpecification, HashCheckingMode, InstallOptions, +}; use uv_dispatch::BuildDispatch; use uv_fs::CWD; use uv_installer::SitePackages; @@ -254,7 +256,8 @@ pub(super) async fn do_sync( // TODO(charlie): These are all default values. We should consider whether we want to make them // optional on the downstream APIs. - let build_constraints = []; + let build_constraints = Constraints::default(); + let build_hasher = HashStrategy::default(); let dry_run = false; // Extract the hashes from the lockfile. @@ -271,7 +274,7 @@ pub(super) async fn do_sync( let build_dispatch = BuildDispatch::new( &client, cache, - &build_constraints, + build_constraints, venv.interpreter(), index_locations, &flat_index, @@ -283,6 +286,7 @@ pub(super) async fn do_sync( build_isolation, link_mode, build_options, + &build_hasher, exclude_newer, sources, concurrency, diff --git a/crates/uv/src/commands/venv.rs b/crates/uv/src/commands/venv.rs index 2967f72bf053..d452cab144be 100644 --- a/crates/uv/src/commands/venv.rs +++ b/crates/uv/src/commands/venv.rs @@ -16,8 +16,8 @@ use uv_auth::store_credentials_from_url; use uv_cache::Cache; use uv_client::{BaseClientBuilder, Connectivity, FlatIndexClient, RegistryClientBuilder}; use uv_configuration::{ - BuildOptions, Concurrency, ConfigSettings, IndexStrategy, KeyringProviderType, NoBinary, - NoBuild, SourceStrategy, TrustedHost, + BuildOptions, Concurrency, ConfigSettings, Constraints, IndexStrategy, KeyringProviderType, + NoBinary, NoBuild, SourceStrategy, TrustedHost, }; use uv_dispatch::BuildDispatch; use uv_fs::{Simplified, CWD}; @@ -300,7 +300,8 @@ async fn venv_impl( let state = SharedState::default(); // For seed packages, assume a bunch of default settings are sufficient. - let build_constraints = []; + let build_constraints = Constraints::default(); + let build_hasher = HashStrategy::default(); let config_settings = ConfigSettings::default(); let sources = SourceStrategy::Disabled; @@ -311,7 +312,7 @@ async fn venv_impl( let build_dispatch = BuildDispatch::new( &client, cache, - &build_constraints, + build_constraints, interpreter, index_locations, &flat_index, @@ -323,6 +324,7 @@ async fn venv_impl( BuildIsolation::Isolated, link_mode, &build_options, + &build_hasher, exclude_newer, sources, concurrency, diff --git a/crates/uv/tests/pip_install.rs b/crates/uv/tests/pip_install.rs index 7b7384ceccb1..df8c966f7bf9 100644 --- a/crates/uv/tests/pip_install.rs +++ b/crates/uv/tests/pip_install.rs @@ -5132,14 +5132,14 @@ fn require_hashes_editable() -> Result<()> { ----- stdout ----- ----- stderr ----- - error: In `--require-hashes` mode, all requirement must have a hash, but none were provided for: file://[WORKSPACE]/scripts/packages/black_editable[d] + error: In `--require-hashes` mode, all requirements must have a hash, but none were provided for: file://[WORKSPACE]/scripts/packages/black_editable[d] "### ); Ok(()) } -/// If a hash is only included as a constraint, that's not good enough for `--require-hashes`. +/// If a hash is only included as a constraint, that's good enough for `--require-hashes`. #[test] fn require_hashes_constraint() -> Result<()> { let context = TestContext::new("3.12"); @@ -5155,19 +5155,25 @@ fn require_hashes_constraint() -> Result<()> { uv_snapshot!(context.pip_install() .arg("-r") .arg(requirements_txt.path()) + .arg("--no-deps") .arg("--require-hashes") .arg("-c") .arg(constraints_txt.path()), @r###" - success: false - exit_code: 2 + success: true + exit_code: 0 ----- stdout ----- ----- stderr ----- - error: In `--require-hashes` mode, all requirement must have a hash, but none were provided for: anyio==4.0.0 + Resolved 1 package in [TIME] + Prepared 1 package in [TIME] + Installed 1 package in [TIME] + + anyio==4.0.0 "### ); // Include the hash in the requirements file, but pin the version in the constraint file. + let context = TestContext::new("3.12"); + let requirements_txt = context.temp_dir.child("requirements.txt"); requirements_txt.write_str( "anyio --hash=sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f", @@ -5180,6 +5186,7 @@ fn require_hashes_constraint() -> Result<()> { uv_snapshot!(context.pip_install() .arg("-r") .arg(requirements_txt.path()) + .arg("--no-deps") .arg("--require-hashes") .arg("-c") .arg(constraints_txt.path()), @r###" @@ -5188,7 +5195,77 @@ fn require_hashes_constraint() -> Result<()> { ----- stdout ----- ----- stderr ----- - error: In `--require-hashes` mode, all requirement must have their versions pinned with `==`, but found: anyio + error: In `--require-hashes` mode, all requirements must have their versions pinned with `==`, but found: anyio + "### + ); + + // Include the wrong hash in the requirements file, but the right hash in constraints. This + // should fail. + let context = TestContext::new("3.12"); + + let requirements_txt = context.temp_dir.child("requirements.txt"); + requirements_txt.write_str( + "anyio==4.0.0 --hash=sha256:afdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f", + )?; + + let constraints_txt = context.temp_dir.child("constraints.txt"); + constraints_txt.write_str("anyio==4.0.0 --hash=sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f")?; + + // Install the editable packages. + uv_snapshot!(context.pip_install() + .arg("-r") + .arg(requirements_txt.path()) + .arg("--no-deps") + .arg("--require-hashes") + .arg("-c") + .arg(constraints_txt.path()), @r###" + success: false + exit_code: 2 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + error: Failed to prepare distributions + Caused by: Failed to fetch wheel: anyio==4.0.0 + Caused by: Hash mismatch for `anyio==4.0.0` + + Expected: + sha256:afdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f + + Computed: + sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f + "### + ); + + // Include the right hash in the requirements file, but the wrong hash in constraints. This + // should succeed. + let context = TestContext::new("3.12"); + + let requirements_txt = context.temp_dir.child("requirements.txt"); + requirements_txt.write_str( + "anyio==4.0.0 --hash=sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f", + )?; + + let constraints_txt = context.temp_dir.child("constraints.txt"); + constraints_txt.write_str("anyio==4.0.0 --hash=sha256:afdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f")?; + + // Install the editable packages. + uv_snapshot!(context.pip_install() + .arg("-r") + .arg(requirements_txt.path()) + .arg("--no-deps") + .arg("--require-hashes") + .arg("-c") + .arg(constraints_txt.path()), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 1 package in [TIME] + Prepared 1 package in [TIME] + Installed 1 package in [TIME] + + anyio==4.0.0 "### ); @@ -5306,7 +5383,7 @@ fn require_hashes_override() -> Result<()> { ----- stdout ----- ----- stderr ----- - error: In `--require-hashes` mode, all requirement must have a hash, but none were provided for: anyio==4.0.0 + error: In `--require-hashes` mode, all requirements must have a hash, but none were provided for: anyio==4.0.0 "### ); @@ -5331,7 +5408,7 @@ fn require_hashes_override() -> Result<()> { ----- stdout ----- ----- stderr ----- - error: In `--require-hashes` mode, all requirement must have their versions pinned with `==`, but found: anyio + error: In `--require-hashes` mode, all requirements must have their versions pinned with `==`, but found: anyio "### ); @@ -5401,17 +5478,21 @@ fn verify_hashes_missing_version() -> Result<()> { # via anyio "})?; - // Raise an error. uv_snapshot!(context.pip_install() .arg("-r") .arg("requirements.txt") .arg("--verify-hashes"), @r###" - success: false - exit_code: 2 + success: true + exit_code: 0 ----- stdout ----- ----- stderr ----- - error: In `--verify-hashes` mode, all requirement must have their versions pinned with `==`, but found: anyio + Resolved 3 packages in [TIME] + Prepared 3 packages in [TIME] + Installed 3 packages in [TIME] + + anyio==4.3.0 + + idna==3.6 + + sniffio==1.3.1 "### ); diff --git a/crates/uv/tests/pip_sync.rs b/crates/uv/tests/pip_sync.rs index bf6ed2b9d539..181c8fda2907 100644 --- a/crates/uv/tests/pip_sync.rs +++ b/crates/uv/tests/pip_sync.rs @@ -3446,7 +3446,7 @@ fn require_hashes_missing_hash() -> Result<()> { ----- stdout ----- ----- stderr ----- - error: In `--require-hashes` mode, all requirement must have a hash, but none were provided for: anyio==4.0.0 + error: In `--require-hashes` mode, all requirements must have a hash, but none were provided for: anyio==4.0.0 "### ); @@ -3487,7 +3487,7 @@ fn require_hashes_missing_version() -> Result<()> { ----- stdout ----- ----- stderr ----- - error: In `--require-hashes` mode, all requirement must have their versions pinned with `==`, but found: anyio + error: In `--require-hashes` mode, all requirements must have their versions pinned with `==`, but found: anyio "### ); @@ -3528,7 +3528,7 @@ fn require_hashes_invalid_operator() -> Result<()> { ----- stdout ----- ----- stderr ----- - error: In `--require-hashes` mode, all requirement must have their versions pinned with `==`, but found: anyio>4.0.0 + error: In `--require-hashes` mode, all requirements must have their versions pinned with `==`, but found: anyio>4.0.0 "### ); @@ -4277,7 +4277,7 @@ fn require_hashes_editable() -> Result<()> { ----- stdout ----- ----- stderr ----- - error: In `--require-hashes` mode, all requirement must have a hash, but none were provided for: file://[WORKSPACE]/scripts/packages/black_editable[d] + error: In `--require-hashes` mode, all requirements must have a hash, but none were provided for: file://[WORKSPACE]/scripts/packages/black_editable[d] "### ); @@ -4301,7 +4301,7 @@ fn require_hashes_repeated_dependency() -> Result<()> { ----- stdout ----- ----- stderr ----- - error: In `--require-hashes` mode, all requirement must have their versions pinned with `==`, but found: anyio + error: In `--require-hashes` mode, all requirements must have their versions pinned with `==`, but found: anyio "### ); @@ -4318,7 +4318,7 @@ fn require_hashes_repeated_dependency() -> Result<()> { ----- stdout ----- ----- stderr ----- - error: In `--require-hashes` mode, all requirement must have their versions pinned with `==`, but found: anyio + error: In `--require-hashes` mode, all requirements must have their versions pinned with `==`, but found: anyio "### ); @@ -5052,7 +5052,7 @@ fn require_hashes_url_other_fragment() -> Result<()> { ----- stdout ----- ----- stderr ----- - error: In `--require-hashes` mode, all requirement must have a hash, but none were provided for: iniconfig @ https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl#foo=bar + error: In `--require-hashes` mode, all requirements must have a hash, but none were provided for: iniconfig @ https://files.pythonhosted.org/packages/ef/a6/62565a6e1cf69e10f5727360368e451d4b7f58beeac6173dc9db836a5b46/iniconfig-2.0.0-py3-none-any.whl#foo=bar "### );