Skip to content

Commit

Permalink
Use distribution database and index for all pre-resolution phases
Browse files Browse the repository at this point in the history
  • Loading branch information
charliermarsh committed Apr 2, 2024
1 parent dfdcce6 commit 5ee0ec4
Show file tree
Hide file tree
Showing 6 changed files with 89 additions and 54 deletions.
10 changes: 10 additions & 0 deletions crates/distribution-types/src/buildable.rs
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,16 @@ pub enum SourceUrl<'a> {
Path(PathSourceUrl<'a>),
}

impl<'a> SourceUrl<'a> {
pub fn url(&self) -> &Url {
match self {
Self::Direct(dist) => dist.url,
Self::Git(dist) => dist.url,
Self::Path(dist) => dist.url,
}
}
}

impl std::fmt::Display for SourceUrl<'_> {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Expand Down
6 changes: 3 additions & 3 deletions crates/uv-requirements/src/lookahead.rs
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,8 @@ impl<'a, Context: BuildContext + Send + Sync> LookaheadResolver<'a, Context> {
// Fetch the metadata for the distribution.
let requires_dist = {
// If the metadata is already in the index, return it.
if let Some(metadata) = self.index.get_metadata(&dist.package_id()) {
let id = dist.package_id();
if let Some(metadata) = self.index.get_metadata(&id) {
metadata.requires_dist.clone()
} else {
// Run the PEP 517 build process to extract metadata from the source distribution.
Expand All @@ -149,8 +150,7 @@ impl<'a, Context: BuildContext + Send + Sync> LookaheadResolver<'a, Context> {
})?;

// Insert the metadata into the index.
self.index
.insert_metadata(dist.package_id(), metadata.clone());
self.index.insert_metadata(id, metadata.clone());

// Insert the redirect into the index.
if let Some(precise) = precise {
Expand Down
96 changes: 57 additions & 39 deletions crates/uv-requirements/src/unnamed.rs
Original file line number Diff line number Diff line change
@@ -1,72 +1,76 @@
use std::borrow::Cow;
use std::path::Path;
use std::str::FromStr;
use std::sync::Arc;

use anyhow::{Context, Result};
use configparser::ini::Ini;
use futures::{StreamExt, TryStreamExt};
use serde::Deserialize;
use tracing::debug;

use cache_key::CanonicalUrl;
use distribution_filename::{SourceDistFilename, WheelFilename};
use distribution_types::{
BuildableSource, DirectSourceUrl, GitSourceUrl, PathSourceUrl, RemoteSource, SourceUrl,
BuildableSource, DirectSourceUrl, GitSourceUrl, PackageId, PathSourceUrl, RemoteSource,
SourceUrl,
};
use pep508_rs::{
Requirement, RequirementsTxtRequirement, Scheme, UnnamedRequirement, VersionOrUrl,
};
use pypi_types::Metadata10;
use uv_client::RegistryClient;
use uv_distribution::{Reporter, SourceDistributionBuilder};
use uv_distribution::{DistributionDatabase, Reporter};
use uv_normalize::PackageName;
use uv_resolver::InMemoryIndex;
use uv_types::BuildContext;

/// Like [`RequirementsSpecification`], but with concrete names for all requirements.
pub struct NamedRequirementsResolver {
pub struct NamedRequirementsResolver<'a, Context: BuildContext + Send + Sync> {
/// The requirements for the project.
requirements: Vec<RequirementsTxtRequirement>,
/// The reporter to use when building source distributions.
reporter: Option<Arc<dyn Reporter>>,
/// The in-memory index for resolving dependencies.
index: &'a InMemoryIndex,
/// The database for fetching and building distributions.
database: DistributionDatabase<'a, Context>,
}

impl NamedRequirementsResolver {
impl<'a, Context: BuildContext + Send + Sync> NamedRequirementsResolver<'a, Context> {
/// Instantiate a new [`NamedRequirementsResolver`] for a given set of requirements.
pub fn new(requirements: Vec<RequirementsTxtRequirement>) -> Self {
pub fn new(
requirements: Vec<RequirementsTxtRequirement>,
context: &'a Context,
client: &'a RegistryClient,
index: &'a InMemoryIndex,
) -> Self {
Self {
requirements,
reporter: None,
index,
database: DistributionDatabase::new(client, context),
}
}

/// Set the [`Reporter`] to use for this resolver.
#[must_use]
pub fn with_reporter(self, reporter: impl Reporter + 'static) -> Self {
let reporter: Arc<dyn Reporter> = Arc::new(reporter);
Self {
reporter: Some(reporter),
database: self.database.with_reporter(reporter),
..self
}
}

/// Resolve any unnamed requirements in the specification.
pub async fn resolve<T: BuildContext>(
self,
context: &T,
client: &RegistryClient,
) -> Result<Vec<Requirement>> {
futures::stream::iter(self.requirements)
pub async fn resolve(self) -> Result<Vec<Requirement>> {
let Self {
requirements,
index,
database,
} = self;
futures::stream::iter(requirements)
.map(|requirement| async {
match requirement {
RequirementsTxtRequirement::Pep508(requirement) => Ok(requirement),
RequirementsTxtRequirement::Unnamed(requirement) => {
Self::resolve_requirement(
requirement,
context,
client,
self.reporter.clone(),
)
.await
Self::resolve_requirement(requirement, index, &database).await
}
}
})
Expand All @@ -76,11 +80,10 @@ impl NamedRequirementsResolver {
}

/// Infer the package name for a given "unnamed" requirement.
async fn resolve_requirement<T: BuildContext>(
async fn resolve_requirement(
requirement: UnnamedRequirement,
context: &T,
client: &RegistryClient,
reporter: Option<Arc<dyn Reporter>>,
index: &InMemoryIndex,
database: &DistributionDatabase<'a, Context>,
) -> Result<Requirement> {
// If the requirement is a wheel, extract the package name from the wheel filename.
//
Expand Down Expand Up @@ -231,20 +234,35 @@ impl NamedRequirementsResolver {
}
};

// Run the PEP 517 build process to extract metadata from the source distribution.
let builder = if let Some(reporter) = reporter {
SourceDistributionBuilder::new(client, context).with_reporter(reporter)
} else {
SourceDistributionBuilder::new(client, context)
};
// Fetch the metadata for the distribution.
let name = {
// If the metadata is already in the index, return it.
let id = PackageId::from_url(source.url());
if let Some(metadata) = index.get_metadata(&id) {
metadata.name.clone()
} else {
// Run the PEP 517 build process to extract metadata from the source distribution.
let (metadata, precise) = database
.build_wheel_metadata(&BuildableSource::Url(source))
.await
.context("Failed to build source distribution")?;

let metadata = builder
.download_and_build_metadata(&BuildableSource::Url(source))
.await
.context("Failed to build source distribution")?;
let name = metadata.name.clone();

// Insert the metadata into the index.
index.insert_metadata(id, metadata);

// Insert the redirect into the index.
if let Some(precise) = precise {
index.insert_redirect(CanonicalUrl::new(&requirement.url), precise);
}

name
}
};

Ok(Requirement {
name: metadata.name,
name,
extras: requirement.extras,
version_or_url: Some(VersionOrUrl::Url(requirement.url)),
marker: requirement.marker,
Expand Down
13 changes: 9 additions & 4 deletions crates/uv/src/commands/pip_compile.rs
Original file line number Diff line number Diff line change
Expand Up @@ -263,10 +263,15 @@ pub(crate) async fn pip_compile(
// Resolve the requirements from the provided sources.
let requirements = {
// Convert from unnamed to named requirements.
let mut requirements = NamedRequirementsResolver::new(requirements)
.with_reporter(ResolverReporter::from(printer))
.resolve(&build_dispatch, &client)
.await?;
let mut requirements = NamedRequirementsResolver::new(
requirements,
&build_dispatch,
&client,
&top_level_index,
)
.with_reporter(ResolverReporter::from(printer))
.resolve()
.await?;

// Resolve any source trees into requirements.
if !source_trees.is_empty() {
Expand Down
9 changes: 5 additions & 4 deletions crates/uv/src/commands/pip_install.rs
Original file line number Diff line number Diff line change
Expand Up @@ -244,10 +244,11 @@ pub(crate) async fn pip_install(
// Resolve the requirements from the provided sources.
let requirements = {
// Convert from unnamed to named requirements.
let mut requirements = NamedRequirementsResolver::new(requirements)
.with_reporter(ResolverReporter::from(printer))
.resolve(&resolve_dispatch, &client)
.await?;
let mut requirements =
NamedRequirementsResolver::new(requirements, &resolve_dispatch, &client, &index)
.with_reporter(ResolverReporter::from(printer))
.resolve()
.await?;

// Resolve any source trees into requirements.
if !source_trees.is_empty() {
Expand Down
9 changes: 5 additions & 4 deletions crates/uv/src/commands/pip_sync.rs
Original file line number Diff line number Diff line change
Expand Up @@ -194,10 +194,11 @@ pub(crate) async fn pip_sync(
// Convert from unnamed to named requirements.
let requirements = {
// Convert from unnamed to named requirements.
let mut requirements = NamedRequirementsResolver::new(requirements)
.with_reporter(ResolverReporter::from(printer))
.resolve(&build_dispatch, &client)
.await?;
let mut requirements =
NamedRequirementsResolver::new(requirements, &build_dispatch, &client, &index)
.with_reporter(ResolverReporter::from(printer))
.resolve()
.await?;

// Resolve any source trees into requirements.
if !source_trees.is_empty() {
Expand Down

0 comments on commit 5ee0ec4

Please sign in to comment.