Skip to content

Commit

Permalink
Gracefully handle non-existent packages in local indexes (#4545)
Browse files Browse the repository at this point in the history
## Summary

Ensures that local indexes can be used as `--extra-index-url` by
gracefully handling "404" errors.

Closes #4540.
  • Loading branch information
charliermarsh authored Jun 26, 2024
1 parent d7f195f commit a5b5856
Show file tree
Hide file tree
Showing 3 changed files with 93 additions and 15 deletions.
4 changes: 4 additions & 0 deletions crates/uv-client/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,10 @@ pub enum ErrorKind {
#[error("Package `{0}` was not found in the registry.")]
PackageNotFound(String),

/// The package was not found in the local (file-based) index.
#[error("Package `{0}` was not found in the local index.")]
FileNotFound(String),

/// The metadata file could not be parsed.
#[error("Couldn't parse metadata of {0} from {1}")]
MetadataParseError(
Expand Down
50 changes: 39 additions & 11 deletions crates/uv-client/src/registry_client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,12 @@ pub struct RegistryClient {
timeout: u64,
}

#[derive(Debug)]
enum IndexError {
Remote(CachedClientError<Error>),
Local(Error),
}

impl RegistryClient {
/// Return the [`CachedClient`] used by this client.
pub fn cached_client(&self) -> &CachedClient {
Expand Down Expand Up @@ -229,8 +235,19 @@ impl RegistryClient {
break;
}
}
Err(CachedClientError::Client(err)) => match err.into_kind() {
Err(IndexError::Local(err)) => {
match err.into_kind() {
// The package could not be found in the local index.
ErrorKind::FileNotFound(_) => continue,

other => return Err(other.into()),
}
}
Err(IndexError::Remote(CachedClientError::Client(err))) => match err.into_kind() {
// The package is unavailable due to a lack of connectivity.
ErrorKind::Offline(_) => continue,

// The package could not be found in the remote index.
ErrorKind::ReqwestError(err) => {
if err.status() == Some(StatusCode::NOT_FOUND)
|| err.status() == Some(StatusCode::UNAUTHORIZED)
Expand All @@ -240,9 +257,10 @@ impl RegistryClient {
}
return Err(ErrorKind::from(err).into());
}

other => return Err(other.into()),
},
Err(CachedClientError::Callback(err)) => return Err(err),
Err(IndexError::Remote(CachedClientError::Callback(err))) => return Err(err),
};
}

Expand All @@ -266,7 +284,7 @@ impl RegistryClient {
&self,
package_name: &PackageName,
index: &IndexUrl,
) -> Result<Result<OwnedArchive<SimpleMetadata>, CachedClientError<Error>>, Error> {
) -> Result<Result<OwnedArchive<SimpleMetadata>, IndexError>, Error> {
// Format the URL for PyPI.
let mut url: Url = index.clone().into();
url.path_segments_mut()
Expand Down Expand Up @@ -298,7 +316,7 @@ impl RegistryClient {
};

if matches!(index, IndexUrl::Path(_)) {
self.fetch_local_index(package_name, &url).await.map(Ok)
self.fetch_local_index(package_name, &url).await
} else {
self.fetch_remote_index(package_name, &url, &cache_entry, cache_control)
.await
Expand All @@ -312,7 +330,7 @@ impl RegistryClient {
url: &Url,
cache_entry: &CacheEntry,
cache_control: CacheControl,
) -> Result<Result<OwnedArchive<SimpleMetadata>, CachedClientError<Error>>, Error> {
) -> Result<Result<OwnedArchive<SimpleMetadata>, IndexError>, Error> {
let simple_request = self
.uncached_client()
.get(url.clone())
Expand Down Expand Up @@ -367,7 +385,8 @@ impl RegistryClient {
cache_control,
parse_simple_response,
)
.await;
.await
.map_err(IndexError::Remote);
Ok(result)
}

Expand All @@ -377,16 +396,25 @@ impl RegistryClient {
&self,
package_name: &PackageName,
url: &Url,
) -> Result<OwnedArchive<SimpleMetadata>, Error> {
) -> Result<Result<OwnedArchive<SimpleMetadata>, IndexError>, Error> {
let path = url
.to_file_path()
.map_err(|()| ErrorKind::NonFileUrl(url.clone()))?
.join("index.html");
let text = fs_err::tokio::read_to_string(&path)
.await
.map_err(ErrorKind::from)?;
let text = match fs_err::tokio::read_to_string(&path).await {
Ok(text) => text,
Err(err) if err.kind() == std::io::ErrorKind::NotFound => {
return Ok(Err(IndexError::Local(Error::from(
ErrorKind::FileNotFound(package_name.to_string()),
))));
}
Err(err) => {
return Err(Error::from(ErrorKind::Io(err)));
}
};
let metadata = SimpleMetadata::from_html(&text, package_name, url)?;
OwnedArchive::from_unarchived(&metadata)
let metadata = OwnedArchive::from_unarchived(&metadata)?;
Ok(Ok(metadata))
}

/// Fetch the metadata for a remote wheel file.
Expand Down
54 changes: 50 additions & 4 deletions crates/uv/tests/pip_install.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5625,7 +5625,7 @@ fn local_index_absolute() -> Result<()> {
<meta name="pypi:repository-version" content="1.1" />
</head>
<body>
<h1>Links for example-a-961b4c22</h1>
<h1>Links for tqdm</h1>
<a
href="{}/tqdm-1000.0.0-py3-none-any.whl"
data-requires-python=">=3.8"
Expand Down Expand Up @@ -5676,7 +5676,7 @@ fn local_index_relative() -> Result<()> {
<meta name="pypi:repository-version" content="1.1" />
</head>
<body>
<h1>Links for example-a-961b4c22</h1>
<h1>Links for tqdm</h1>
<a
href="{}/tqdm-1000.0.0-py3-none-any.whl"
data-requires-python=">=3.8"
Expand Down Expand Up @@ -5727,7 +5727,7 @@ fn local_index_requirements_txt_absolute() -> Result<()> {
<meta name="pypi:repository-version" content="1.1" />
</head>
<body>
<h1>Links for example-a-961b4c22</h1>
<h1>Links for tqdm</h1>
<a
href="{}/tqdm-1000.0.0-py3-none-any.whl"
data-requires-python=">=3.8"
Expand Down Expand Up @@ -5783,7 +5783,7 @@ fn local_index_requirements_txt_relative() -> Result<()> {
<meta name="pypi:repository-version" content="1.1" />
</head>
<body>
<h1>Links for example-a-961b4c22</h1>
<h1>Links for tqdm</h1>
<a
href="{}/tqdm-1000.0.0-py3-none-any.whl"
data-requires-python=">=3.8"
Expand Down Expand Up @@ -5820,3 +5820,49 @@ fn local_index_requirements_txt_relative() -> Result<()> {

Ok(())
}

/// Resolve against a local directory laid out as a PEP 503-compatible index, falling back to
/// the default index.
#[test]
fn local_index_fallback() -> Result<()> {
let context = TestContext::new("3.12");

let root = context.temp_dir.child("simple-html");
fs_err::create_dir_all(&root)?;

let tqdm = root.child("tqdm");
fs_err::create_dir_all(&tqdm)?;

let index = tqdm.child("index.html");
index.write_str(
r#"
<!DOCTYPE html>
<html>
<head>
<meta name="pypi:repository-version" content="1.1" />
</head>
<body>
<h1>Links for tqdm</h1>
</body>
</html>
"#,
)?;

uv_snapshot!(context.filters(), context.pip_install()
.arg("iniconfig")
.arg("--extra-index-url")
.arg(Url::from_directory_path(root).unwrap().as_str()), @r###"
success: true
exit_code: 0
----- stdout -----
----- stderr -----
Resolved 1 package in [TIME]
Prepared 1 package in [TIME]
Installed 1 package in [TIME]
+ iniconfig==2.0.0
"###
);

Ok(())
}

0 comments on commit a5b5856

Please sign in to comment.