Skip to content

Commit

Permalink
Respect resolved Git SHAs in uv lock (#3956)
Browse files Browse the repository at this point in the history
## Summary

This PR ensures that if a lockfile already contains a resolved reference
(e.g., you locked with `main` previously, and it locked to a specific
commit), and you run `uv lock`, we use the same SHA, even if it's not
the latest SHA for that tag. This avoids upgrading Git dependencies
without `--upgrade`.

Closes #3920.
  • Loading branch information
charliermarsh committed Jun 1, 2024
1 parent b7d77c0 commit c04a95e
Show file tree
Hide file tree
Showing 6 changed files with 251 additions and 66 deletions.
4 changes: 3 additions & 1 deletion crates/uv-git/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,9 @@ use std::str::FromStr;
use url::Url;

pub use crate::git::GitReference;
pub use crate::resolver::{GitResolver, GitResolverError, RepositoryReference};
pub use crate::resolver::{
GitResolver, GitResolverError, RepositoryReference, ResolvedRepositoryReference,
};
pub use crate::sha::{GitOid, GitSha, OidParseError};
pub use crate::source::{Fetch, GitSource, Reporter};

Expand Down
20 changes: 19 additions & 1 deletion crates/uv-git/src/resolver.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,15 @@ pub enum GitResolverError {
pub struct GitResolver(Arc<DashMap<RepositoryReference, GitSha>>);

impl GitResolver {
/// Initialize a [`GitResolver`] with a set of resolved references.
pub fn from_refs(refs: Vec<ResolvedRepositoryReference>) -> Self {
Self(Arc::new(
refs.into_iter()
.map(|ResolvedRepositoryReference { reference, sha }| (reference, sha))
.collect(),
))
}

/// Returns the [`GitSha`] for the given [`RepositoryReference`], if it exists.
pub fn get(&self, reference: &RepositoryReference) -> Option<Ref<RepositoryReference, GitSha>> {
self.0.get(reference)
Expand Down Expand Up @@ -136,11 +145,20 @@ impl GitResolver {
}
}

#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct ResolvedRepositoryReference {
/// An abstract reference to a Git repository, including the URL and the commit (e.g., a branch,
/// tag, or revision).
pub reference: RepositoryReference,
/// The precise commit SHA of the reference.
pub sha: GitSha,
}

#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct RepositoryReference {
/// The URL of the Git repository, with any query parameters and fragments removed.
pub url: RepositoryUrl,
/// The reference to the commit to use, which could be a branch, tag or revision.
/// The reference to the commit to use, which could be a branch, tag, or revision.
pub reference: GitReference,
}

Expand Down
57 changes: 35 additions & 22 deletions crates/uv-requirements/src/upgrade.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,17 @@ use requirements_txt::RequirementsTxt;
use uv_client::{BaseClientBuilder, Connectivity};
use uv_configuration::Upgrade;
use uv_distribution::ProjectWorkspace;
use uv_git::ResolvedRepositoryReference;
use uv_resolver::{Lock, Preference, PreferenceError};

#[derive(Debug, Default)]
pub struct LockedRequirements {
/// The pinned versions from the lockfile.
pub preferences: Vec<Preference>,
/// The pinned Git SHAs from the lockfile.
pub git: Vec<ResolvedRepositoryReference>,
}

/// Load the preferred requirements from an existing `requirements.txt`, applying the upgrade strategy.
pub async fn read_requirements_txt(
output_file: Option<&Path>,
Expand Down Expand Up @@ -58,10 +67,10 @@ pub async fn read_requirements_txt(
pub async fn read_lockfile(
project: &ProjectWorkspace,
upgrade: &Upgrade,
) -> Result<Vec<Preference>> {
) -> Result<LockedRequirements> {
// As an optimization, skip reading the lockfile is we're upgrading all packages anyway.
if upgrade.is_all() {
return Ok(Vec::new());
return Ok(LockedRequirements::default());
}

// If an existing lockfile exists, build up a set of preferences.
Expand All @@ -71,32 +80,36 @@ pub async fn read_lockfile(
Ok(lock) => lock,
Err(err) => {
eprint!("Failed to parse lockfile; ignoring locked requirements: {err}");
return Ok(Vec::new());
return Ok(LockedRequirements::default());
}
},
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
return Ok(Vec::new());
return Ok(LockedRequirements::default());
}
Err(err) => return Err(err.into()),
};

// Map each entry in the lockfile to a preference.
let preferences: Vec<Preference> = lock
.distributions()
.iter()
.map(Preference::from_lock)
.collect();
let mut preferences = Vec::new();
let mut git = Vec::new();

// Apply the upgrade strategy to the requirements.
Ok(match upgrade {
// Respect all pinned versions from the existing lockfile.
Upgrade::None => preferences,
// Ignore all pinned versions from the existing lockfile.
Upgrade::All => vec![],
// Ignore pinned versions for the specified packages.
Upgrade::Packages(packages) => preferences
.into_iter()
.filter(|preference| !packages.contains(preference.name()))
.collect(),
})
for dist in lock.distributions() {
// Skip the distribution if it's not included in the upgrade strategy.
if match upgrade {
Upgrade::None => false,
Upgrade::All => true,
Upgrade::Packages(packages) => packages.contains(dist.name()),
} {
continue;
}

// Map each entry in the lockfile to a preference.
preferences.push(Preference::from_lock(dist));

// Map each entry in the lockfile to a Git SHA.
if let Some(git_ref) = dist.as_git_ref() {
git.push(git_ref);
}
}

Ok(LockedRequirements { preferences, git })
}
22 changes: 21 additions & 1 deletion crates/uv-resolver/src/lock.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ use std::path::{Path, PathBuf};
use std::str::FromStr;

use anyhow::Result;
use cache_key::RepositoryUrl;
use rustc_hash::FxHashMap;
use toml_edit::{value, Array, ArrayOfTables, InlineTable, Item, Table, Value};
use url::Url;
Expand All @@ -21,7 +22,7 @@ use pep440_rs::Version;
use pep508_rs::{MarkerEnvironment, VerbatimUrl};
use platform_tags::{TagCompatibility, TagPriority, Tags};
use pypi_types::{HashDigest, ParsedArchiveUrl, ParsedGitUrl};
use uv_git::{GitReference, GitSha};
use uv_git::{GitReference, GitSha, RepositoryReference, ResolvedRepositoryReference};
use uv_normalize::{ExtraName, PackageName};

use crate::resolution::AnnotatedDist;
Expand Down Expand Up @@ -478,6 +479,25 @@ impl Distribution {
}
best.map(|(_, i)| i)
}

/// Returns the [`PackageName`] of the distribution.
pub fn name(&self) -> &PackageName {
&self.id.name
}

/// Returns the [`ResolvedRepositoryReference`] for the distribution, if it is a Git source.
pub fn as_git_ref(&self) -> Option<ResolvedRepositoryReference> {
match &self.id.source.kind {
SourceKind::Git(git) => Some(ResolvedRepositoryReference {
reference: RepositoryReference {
url: RepositoryUrl::new(&self.id.source.url),
reference: GitReference::from(git.kind.clone()),
},
sha: git.precise,
}),
_ => None,
}
}
}

#[derive(Clone, Debug, Eq, Hash, PartialEq, PartialOrd, Ord, serde::Deserialize)]
Expand Down
8 changes: 5 additions & 3 deletions crates/uv/src/commands/project/lock.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ use uv_dispatch::BuildDispatch;
use uv_distribution::ProjectWorkspace;
use uv_git::GitResolver;
use uv_interpreter::PythonEnvironment;
use uv_requirements::upgrade::read_lockfile;
use uv_requirements::upgrade::{read_lockfile, LockedRequirements};
use uv_resolver::{ExcludeNewer, FlatIndex, InMemoryIndex, Lock, OptionsBuilder};
use uv_types::{BuildIsolation, EmptyInstalledPackages, HashStrategy, InFlight};
use uv_warnings::warn_user;
Expand Down Expand Up @@ -105,7 +105,6 @@ pub(super) async fn do_lock(
let config_settings = ConfigSettings::default();
let extras = ExtrasSpecification::default();
let flat_index = FlatIndex::default();
let git = GitResolver::default();
let in_flight = InFlight::default();
let index = InMemoryIndex::default();
let index_locations = IndexLocations::default();
Expand All @@ -119,7 +118,10 @@ pub(super) async fn do_lock(
let options = OptionsBuilder::new().exclude_newer(exclude_newer).build();

// If an existing lockfile exists, build up a set of preferences.
let preferences = read_lockfile(project, &upgrade).await?;
let LockedRequirements { preferences, git } = read_lockfile(project, &upgrade).await?;

// Create the Git resolver.
let git = GitResolver::from_refs(git);

// Create a build dispatch.
let build_dispatch = BuildDispatch::new(
Expand Down
Loading

0 comments on commit c04a95e

Please sign in to comment.