Recursively allow URL requirements for local dependencies (astral-sh#…

…2702)

## Summary

This is a trimmed-down version of
astral-sh#2684 that only applies to local
source trees for now, which enables workspace-like workflows (whereby
local packages can depend on other local packages at arbitrary depth).

Closes astral-sh#2699.

## Test Plan

Added new tests.

Also cloned this MRE that was shared with me
(https://github.com/timothyjlaurent/uv-poetry-monorepo-mre), and
verified that it was installed without error:

```
❯ cargo run pip install ./uv-poetry-monorepo-mre/app --no-cache
    Finished dev [unoptimized + debuginfo] target(s) in 0.15s
     Running `target/debug/uv pip install ./uv-poetry-monorepo-mre/app --no-cache`
Resolved 4 packages in 1.28s
   Built app @ file:///Users/crmarsh/workspace/uv/uv-poetry-monorepo-mre/app
   Built lib1 @ file:///Users/crmarsh/workspace/uv/uv-poetry-monorepo-mre/lib1
   Built lib2 @ file:///Users/crmarsh/workspace/uv/uv-poetry-monorepo-mre/lib2                                        Downloaded 4 packages in 457ms
Installed 4 packages in 2ms
 + app==0.1.0 (from file:///Users/crmarsh/workspace/uv/uv-poetry-monorepo-mre/app)
 + lib1==0.1.0 (from file:///Users/crmarsh/workspace/uv/uv-poetry-monorepo-mre/lib1)
 + lib2==0.1.0 (from file:///Users/crmarsh/workspace/uv/uv-poetry-monorepo-mre/lib2)
 + ruff==0.3.4
```

crates/uv-requirements/src/lookahead.rs

@@ -1,10 +1,12 @@
+use std::collections::VecDeque;
 use std::sync::Arc;
 
-use anyhow::{Context, Result};
-use futures::{StreamExt, TryStreamExt};
+use anyhow::Result;
+use futures::stream::FuturesUnordered;
+use futures::StreamExt;
 
 use distribution_types::{BuildableSource, Dist};
-use pep508_rs::{Requirement, VersionOrUrl};
+use pep508_rs::{MarkerEnvironment, Requirement, VersionOrUrl};
 use uv_client::RegistryClient;
 use uv_distribution::{Reporter, SourceDistCachedBuilder};
 use uv_types::{BuildContext, RequestedRequirements};
@@ -20,16 +22,16 @@ use uv_types::{BuildContext, RequestedRequirements};
 ///
 /// The lookahead resolver resolves requirements for local dependencies, so that the resolver can
 /// treat them as first-party dependencies for the purpose of analyzing their specifiers.
-pub struct LookaheadResolver<'a> {
+pub struct LookaheadResolver {
     /// The requirements for the project.
-    requirements: &'a [Requirement],
+    requirements: Vec<Requirement>,
     /// The reporter to use when building source distributions.
     reporter: Option<Arc<dyn Reporter>>,
 }
 
-impl<'a> LookaheadResolver<'a> {
-    /// Instantiate a new [`LookaheadResolver`] for a given set of `source_trees`.
-    pub fn new(requirements: &'a [Requirement]) -> Self {
+impl LookaheadResolver {
+    /// Instantiate a new [`LookaheadResolver`] for a given set of requirements.
+    pub fn new(requirements: Vec<Requirement>) -> Self {
         Self {
             requirements,
             reporter: None,
@@ -50,20 +52,37 @@ impl<'a> LookaheadResolver<'a> {
     pub async fn resolve<T: BuildContext>(
         self,
         context: &T,
+        markers: &MarkerEnvironment,
         client: &RegistryClient,
     ) -> Result<Vec<RequestedRequirements>> {
-        let requirements: Vec<_> = futures::stream::iter(self.requirements.iter())
-            .map(|requirement| async { self.lookahead(requirement, context, client).await })
-            .buffered(50)
-            .try_collect()
-            .await?;
-        Ok(requirements.into_iter().flatten().collect())
+        let mut queue = VecDeque::from(self.requirements.clone());
+        let mut results = Vec::new();
+        let mut futures = FuturesUnordered::new();
+
+        while !queue.is_empty() || !futures.is_empty() {
+            while let Some(requirement) = queue.pop_front() {
+                futures.push(self.lookahead(requirement, context, client));
+            }
+
+            while let Some(result) = futures.next().await {
+                if let Some(lookahead) = result? {
+                    for requirement in lookahead.requirements() {
+                        if requirement.evaluate_markers(markers, lookahead.extras()) {
+                            queue.push_back(requirement.clone());
+                        }
+                    }
+                    results.push(lookahead);
+                }
+            }
+        }
+
+        Ok(results)
     }
 
     /// Infer the package name for a given "unnamed" requirement.
     async fn lookahead<T: BuildContext>(
         &self,
-        requirement: &Requirement,
+        requirement: Requirement,
         context: &T,
         client: &RegistryClient,
     ) -> Result<Option<RequestedRequirements>> {
@@ -73,7 +92,7 @@ impl<'a> LookaheadResolver<'a> {
         };
 
         // Convert to a buildable distribution.
-        let dist = Dist::from_url(requirement.name.clone(), url.clone())?;
+        let dist = Dist::from_url(requirement.name, url.clone())?;
 
         // Only support source trees (and not, e.g., wheels).
         let Dist::Source(source_dist) = &dist else {
@@ -92,12 +111,11 @@ impl<'a> LookaheadResolver<'a> {
 
         let metadata = builder
             .download_and_build_metadata(&BuildableSource::Dist(source_dist))
-            .await
-            .context("Failed to build source distribution")?;
+            .await?;
 
         // Return the requirements from the metadata.
         Ok(Some(RequestedRequirements::new(
-            requirement.extras.clone(),
+            requirement.extras,
             metadata.requires_dist,
         )))
     }

crates/uv/src/commands/pip_compile.rs

@@ -277,14 +277,8 @@ pub(crate) async fn pip_compile(
         requirements
     };
 
-    // Determine any lookahead requirements.
-    let lookaheads = LookaheadResolver::new(&requirements)
-        .with_reporter(ResolverReporter::from(printer))
-        .resolve(&build_dispatch, &client)
-        .await?;
-
     // Build the editables and add their requirements
-    let editable_metadata = if editables.is_empty() {
+    let editables = if editables.is_empty() {
         Vec::new()
     } else {
         let start = std::time::Instant::now();
@@ -339,14 +333,32 @@ pub(crate) async fn pip_compile(
         editables
     };
 
+    // Determine any lookahead requirements.
+    let lookaheads = LookaheadResolver::new(
+        requirements
+            .iter()
+            .filter(|requirement| requirement.evaluate_markers(&markers, &[]))
+            .chain(editables.iter().flat_map(|(editable, metadata)| {
+                metadata
+                    .requires_dist
+                    .iter()
+                    .filter(|requirement| requirement.evaluate_markers(&markers, &editable.extras))
+            }))
+            .cloned()
+            .collect(),
+    )
+    .with_reporter(ResolverReporter::from(printer))
+    .resolve(&build_dispatch, &markers, &client)
+    .await?;
+
     // Create a manifest of the requirements.
     let manifest = Manifest::new(
         requirements,
         constraints,
         overrides,
         preferences,
         project,
-        editable_metadata,
+        editables,
         // Do not consider any installed packages during compilation
         Exclusions::All,
         lookaheads,

crates/uv/src/commands/pip_install.rs

@@ -16,7 +16,7 @@ use distribution_types::{
 use install_wheel_rs::linker::LinkMode;
 use pep508_rs::{MarkerEnvironment, Requirement};
 use platform_tags::Tags;
-use pypi_types::Yanked;
+use pypi_types::{Metadata23, Yanked};
 use requirements_txt::EditableRequirement;
 use uv_auth::{KeyringProvider, GLOBAL_AUTH_STORE};
 use uv_cache::Cache;
@@ -524,7 +524,7 @@ async fn resolve(
         .collect();
 
     // Map the editables to their metadata.
-    let editables = editables
+    let editables: Vec<(LocalEditable, Metadata23)> = editables
         .iter()
         .map(|built_editable| {
             (
@@ -535,10 +535,22 @@ async fn resolve(
         .collect();
 
     // Determine any lookahead requirements.
-    let lookaheads = LookaheadResolver::new(&requirements)
-        .with_reporter(ResolverReporter::from(printer))
-        .resolve(build_dispatch, client)
-        .await?;
+    let lookaheads = LookaheadResolver::new(
+        requirements
+            .iter()
+            .filter(|requirement| requirement.evaluate_markers(markers, &[]))
+            .chain(editables.iter().flat_map(|(editable, metadata)| {
+                metadata
+                    .requires_dist
+                    .iter()
+                    .filter(|requirement| requirement.evaluate_markers(markers, &editable.extras))
+            }))
+            .cloned()
+            .collect(),
+    )
+    .with_reporter(ResolverReporter::from(printer))
+    .resolve(build_dispatch, markers, client)
+    .await?;
 
     // Create a manifest of the requirements.
     let manifest = Manifest::new(

crates/uv/tests/pip_compile.rs

@@ -6542,6 +6542,92 @@ fn pendulum_no_tzdata_on_windows() -> Result<()> {
     Ok(())
 }
 
+/// Allow URL dependencies recursively for local source trees.
+#[test]
+fn allow_recursive_url_local_path() -> Result<()> {
+    let context = TestContext::new("3.12");
+
+    // Create a standalone library.
+    let lib2 = context.temp_dir.child("lib2");
+    lib2.create_dir_all()?;
+    let pyproject_toml = lib2.child("pyproject.toml");
+    pyproject_toml.write_str(
+        r#"[project]
+name = "lib2"
+version = "0.0.0"
+dependencies = [
+    "idna"
+]
+requires-python = ">3.8"
+"#,
+    )?;
+
+    // Create a library that depends on the standalone library.
+    let lib1 = context.temp_dir.child("lib1");
+    lib1.create_dir_all()?;
+    let pyproject_toml = lib1.child("pyproject.toml");
+    pyproject_toml.write_str(&format!(
+        r#"[project]
+name = "lib1"
+version = "0.0.0"
+dependencies = [
+    "lib2 @ {}"
+]
+requires-python = ">3.8"
+"#,
+        Url::from_directory_path(lib2.path()).unwrap().as_str(),
+    ))?;
+
+    // Create an application that depends on the library.
+    let app = context.temp_dir.child("app");
+    app.create_dir_all()?;
+    let pyproject_toml = app.child("pyproject.toml");
+    pyproject_toml.write_str(&format!(
+        r#"[project]
+name = "example"
+version = "0.0.0"
+dependencies = [
+  "anyio",
+  "lib1 @ {}"
+]
+requires-python = ">3.8"
+"#,
+        Url::from_directory_path(lib1.path()).unwrap().as_str(),
+    ))?;
+
+    // Write to a requirements file.
+    let requirements_in = context.temp_dir.child("requirements.in");
+    requirements_in.write_str("./app")?;
+
+    uv_snapshot!(context.filters(), context.compile()
+        .arg("requirements.in"), @r###"
+    success: true
+    exit_code: 0
+    ----- stdout -----
+    # This file was autogenerated by uv via the following command:
+    #    uv pip compile --cache-dir [CACHE_DIR] --exclude-newer 2024-03-25T00:00:00Z requirements.in
+    anyio==4.3.0
+        # via example
+    example @ ./app
+    idna==3.6
+        # via
+        #   anyio
+        #   lib2
+    lib1 @ file://[TEMP_DIR]/lib1/
+        # via example
+    lib2 @ file://[TEMP_DIR]/lib2/
+        # via lib1
+    sniffio==1.3.1
+        # via anyio
+
+    ----- stderr -----
+    Resolved 6 packages in [TIME]
+    "###
+    );
+
+    Ok(())
+}
+
 /// Allow pre-releases for dependencies of source path requirements.
 #[test]
 fn pre_release_path_requirement() -> Result<()> {