Skip to content

Commit

Permalink
fix: Always use sparse index
Browse files Browse the repository at this point in the history
  • Loading branch information
epage committed Nov 1, 2023
1 parent 61df7d9 commit dbe514d
Show file tree
Hide file tree
Showing 11 changed files with 733 additions and 58 deletions.
597 changes: 568 additions & 29 deletions Cargo.lock

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,8 @@ vendored-openssl = ["git2/vendored-openssl"]

[dependencies]
cargo_metadata = "0.18"
crates-index = "0.19"
tame-index = "0.5"
reqwest = { version = "0.11", default-features = false, features = ["blocking", "rustls-tls", "gzip"] }
git2 = { version = "0.17.2", default-features = false }
toml_edit = "0.20.7"
toml = "0.8.6"
Expand Down
3 changes: 2 additions & 1 deletion src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ process_error_from!(anyhow::Error);
process_error_from!(std::io::Error);
process_error_from!(semver::Error);
process_error_from!(ignore::Error);
process_error_from!(crates_index::Error);
process_error_from!(tame_index::Error);
process_error_from!(reqwest::Error);
process_error_from!(cargo_metadata::Error);
process_error_from!(toml::ser::Error);
process_error_from!(toml_edit::ser::Error);
Expand Down
27 changes: 17 additions & 10 deletions src/ops/cargo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ pub fn publish(
}

pub fn wait_for_publish(
index: &mut crates_index::Index,
index: &mut crate::ops::index::CratesIoIndex,
name: &str,
version: &str,
timeout: std::time::Duration,
Expand All @@ -122,9 +122,7 @@ pub fn wait_for_publish(
let sleep_time = std::time::Duration::from_secs(1);
let mut logged = false;
loop {
if let Err(e) = index.update() {
log::debug!("crate index update failed with {}", e);
}
index.update_krate(name);
if is_published(index, name, version) {
break;
} else if timeout < now.elapsed() {
Expand All @@ -145,12 +143,21 @@ pub fn wait_for_publish(
Ok(())
}

pub fn is_published(index: &crates_index::Index, name: &str, version: &str) -> bool {
let crate_data = index.crate_(name);
crate_data
.iter()
.flat_map(|c| c.versions().iter())
.any(|v| v.version() == version)
pub fn is_published(
index: &mut crate::ops::index::CratesIoIndex,
name: &str,
version: &str,
) -> bool {
match index.has_krate_version(name, version) {
Ok(has_krate_version) => has_krate_version.unwrap_or(false),
Err(err) => {
// For both http and git indices, this _might_ be an error that goes away in
// a future call, but at least printing out something should give the user
// an indication something is amiss
log::warn!("failed to read metadata for {name}: {err:#}");
false
}
}
}

pub fn set_workspace_version(
Expand Down
121 changes: 121 additions & 0 deletions src/ops/index.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
use tame_index::krate::IndexKrate;

pub struct CratesIoIndex {
index: RemoteIndex,
cache: std::collections::HashMap<String, Option<IndexKrate>>,
}

impl CratesIoIndex {
#[inline]
pub fn open() -> Result<Self, crate::error::CliError> {
Ok(Self {
index: RemoteIndex::open()?,
cache: Default::default(),
})
}

/// Determines if the specified crate exists in the crates.io index
#[inline]
pub fn has_krate(&mut self, name: &str) -> Result<bool, crate::error::CliError> {
Ok(self.krate(name)?.map(|_| true).unwrap_or(false))
}

/// Determines if the specified crate version exists in the crates.io index
#[inline]
pub fn has_krate_version(
&mut self,
name: &str,
version: &str,
) -> Result<Option<bool>, crate::error::CliError> {
let krate = self.krate(name)?;
Ok(krate.map(|ik| ik.versions.iter().any(|iv| iv.version == version)))
}

#[inline]
pub fn update_krate(&mut self, name: &str) {
self.cache.remove(name);
}

pub(crate) fn krate(
&mut self,
name: &str,
) -> Result<Option<IndexKrate>, crate::error::CliError> {
if let Some(entry) = self.cache.get(name) {
log::trace!("Reusing index for {name}");
return Ok(entry.clone());
}

log::trace!("Downloading index for {name}");
let entry = self.index.krate(name)?;
self.cache.insert(name.to_owned(), entry.clone());
Ok(entry)
}
}

pub struct RemoteIndex {
index: tame_index::SparseIndex,
client: reqwest::blocking::Client,
etags: Vec<(String, String)>,
}

impl RemoteIndex {
#[inline]
pub fn open() -> Result<Self, crate::error::CliError> {
let index = tame_index::SparseIndex::new(tame_index::IndexLocation::new(
tame_index::IndexUrl::CratesIoSparse,
))?;
let client = reqwest::blocking::ClientBuilder::new()
.http2_prior_knowledge()
.build()?;

Ok(Self {
index,
client,
etags: Vec::new(),
})
}

pub(crate) fn krate(
&mut self,
name: &str,
) -> Result<Option<IndexKrate>, crate::error::CliError> {
let etag = self
.etags
.iter()
.find_map(|(krate, etag)| (krate == name).then_some(etag.as_str()))
.unwrap_or("");

let krate_name = name.try_into()?;
let req = self.index.make_remote_request(krate_name, Some(etag))?;
let res = self.client.execute(req.try_into()?)?;

// Grab the etag if it exists for future requests
if let Some(etag) = res.headers().get(reqwest::header::ETAG) {
if let Ok(etag) = etag.to_str() {
if let Some(i) = self.etags.iter().position(|(krate, _)| krate == name) {
self.etags[i].1 = etag.to_owned();
} else {
self.etags.push((name.to_owned(), etag.to_owned()));
}
}
}

let mut builder = tame_index::external::http::Response::builder()
.status(res.status())
.version(res.version());

builder
.headers_mut()
.unwrap()
.extend(res.headers().iter().map(|(k, v)| (k.clone(), v.clone())));

let body = res.bytes()?;
let response = builder
.body(body.to_vec())
.map_err(|e| tame_index::Error::from(tame_index::error::HttpError::from(e)))?;

self.index
.parse_remote_response(krate_name, response, false)
.map_err(Into::into)
}
}
1 change: 1 addition & 0 deletions src/ops/mod.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
pub mod cargo;
pub mod cmd;
pub mod git;
pub mod index;
pub mod replace;
pub mod shell;
pub mod version;
4 changes: 2 additions & 2 deletions src/steps/hook.rs
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ pub struct HookStep {
impl HookStep {
pub fn run(&self) -> Result<(), CliError> {
git::git_version()?;
let index = crates_index::Index::new_cargo_default()?;
let mut index = crate::ops::index::CratesIoIndex::open()?;

if self.dry_run {
let _ =
Expand Down Expand Up @@ -87,7 +87,7 @@ impl HookStep {
{
let version = &pkg.initial_version;
if !crate::ops::cargo::is_published(
&index,
&mut index,
crate_name,
&version.full_version_string,
) {
Expand Down
4 changes: 2 additions & 2 deletions src/steps/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ pub fn verify_monotonically_increasing(

pub fn verify_rate_limit(
pkgs: &[plan::PackageRelease],
index: &crates_index::Index,
index: &mut crate::ops::index::CratesIoIndex,
dry_run: bool,
level: log::Level,
) -> Result<bool, crate::error::CliError> {
Expand All @@ -227,7 +227,7 @@ pub fn verify_rate_limit(
for pkg in pkgs {
if pkg.config.registry().is_none() && pkg.config.publish() {
let crate_name = pkg.meta.name.as_str();
if index.crate_(crate_name).is_some() {
if index.has_krate(crate_name)? {
existing += 1;
} else {
new += 1;
Expand Down
14 changes: 9 additions & 5 deletions src/steps/publish.rs
Original file line number Diff line number Diff line change
Expand Up @@ -80,13 +80,16 @@ impl PublishStep {

let mut pkgs = plan::plan(pkgs)?;

let mut index = crates_index::Index::new_cargo_default()?;
let mut index = crate::ops::index::CratesIoIndex::open()?;
for pkg in pkgs.values_mut() {
if pkg.config.registry().is_none() && pkg.config.release() {
let crate_name = pkg.meta.name.as_str();
let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version);
if crate::ops::cargo::is_published(&index, crate_name, &version.full_version_string)
{
if crate::ops::cargo::is_published(
&mut index,
crate_name,
&version.full_version_string,
) {
let _ = crate::ops::shell::warn(format!(
"disabled due to previous publish ({}), skipping {}",
version.full_version_string, crate_name
Expand Down Expand Up @@ -131,7 +134,8 @@ impl PublishStep {
)?;

failed |= !super::verify_metadata(&selected_pkgs, dry_run, log::Level::Error)?;
failed |= !super::verify_rate_limit(&selected_pkgs, &index, dry_run, log::Level::Error)?;
failed |=
!super::verify_rate_limit(&selected_pkgs, &mut index, dry_run, log::Level::Error)?;

// STEP 1: Release Confirmation
super::confirm("Publish", &selected_pkgs, self.no_confirm, dry_run)?;
Expand All @@ -156,7 +160,7 @@ impl PublishStep {
pub fn publish(
ws_meta: &cargo_metadata::Metadata,
pkgs: &[plan::PackageRelease],
index: &mut crates_index::Index,
index: &mut crate::ops::index::CratesIoIndex,
dry_run: bool,
) -> Result<(), CliError> {
for pkg in pkgs {
Expand Down
13 changes: 7 additions & 6 deletions src/steps/release.rs
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ pub struct ReleaseStep {
impl ReleaseStep {
pub fn run(&self) -> Result<(), CliError> {
git::git_version()?;
let mut index = crates_index::Index::new_cargo_default()?;
let mut index = crate::ops::index::CratesIoIndex::open()?;

if self.dry_run {
let _ =
Expand All @@ -73,7 +73,7 @@ impl ReleaseStep {
pkg.bump(level_or_version, self.metadata.as_deref())?;
}
}
if index.crate_(&pkg.meta.name).is_some() {
if index.has_krate(&pkg.meta.name)? {
// Already published, skip it. Use `cargo release owner` for one-time updates
pkg.ensure_owners = false;
}
Expand Down Expand Up @@ -101,7 +101,7 @@ impl ReleaseStep {
&& !explicitly_excluded
{
let version = &pkg.initial_version;
if !cargo::is_published(&index, crate_name, &version.full_version_string) {
if !cargo::is_published(&mut index, crate_name, &version.full_version_string) {
log::debug!(
"enabled {}, v{} is unpublished",
crate_name,
Expand Down Expand Up @@ -155,7 +155,7 @@ impl ReleaseStep {
if pkg.config.publish() && pkg.config.registry().is_none() {
let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version);
let crate_name = pkg.meta.name.as_str();
if !cargo::is_published(&index, crate_name, &version.full_version_string) {
if !cargo::is_published(&mut index, crate_name, &version.full_version_string) {
let _ = crate::ops::shell::warn(format!(
"disabled by user, skipping {} v{} despite being unpublished",
crate_name, version.full_version_string,
Expand Down Expand Up @@ -198,7 +198,7 @@ impl ReleaseStep {
if pkg.config.registry().is_none() {
let version = pkg.planned_version.as_ref().unwrap_or(&pkg.initial_version);
let crate_name = pkg.meta.name.as_str();
if cargo::is_published(&index, crate_name, &version.full_version_string) {
if cargo::is_published(&mut index, crate_name, &version.full_version_string) {
let _ = crate::ops::shell::error(format!(
"{} {} is already published",
crate_name, version.full_version_string
Expand Down Expand Up @@ -231,7 +231,8 @@ impl ReleaseStep {
)?;

failed |= !super::verify_metadata(&selected_pkgs, dry_run, log::Level::Error)?;
failed |= !super::verify_rate_limit(&selected_pkgs, &index, dry_run, log::Level::Error)?;
failed |=
!super::verify_rate_limit(&selected_pkgs, &mut index, dry_run, log::Level::Error)?;

// STEP 1: Release Confirmation
super::confirm("Release", &selected_pkgs, self.no_confirm, dry_run)?;
Expand Down
4 changes: 2 additions & 2 deletions src/steps/replace.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ pub struct ReplaceStep {
impl ReplaceStep {
pub fn run(&self) -> Result<(), CliError> {
git::git_version()?;
let index = crates_index::Index::new_cargo_default()?;
let mut index = crate::ops::index::CratesIoIndex::open()?;

if self.dry_run {
let _ =
Expand Down Expand Up @@ -83,7 +83,7 @@ impl ReplaceStep {
{
let version = &pkg.initial_version;
if !crate::ops::cargo::is_published(
&index,
&mut index,
crate_name,
&version.full_version_string,
) {
Expand Down

0 comments on commit dbe514d

Please sign in to comment.