Skip to content

Commit

Permalink
fix: Temporarily disable deprovisioning (#937)
Browse files Browse the repository at this point in the history
Due to the production issues observed on July 25/26th PST, we will
temporarily disable deprovisioning to prevent any existing logic errors
to impact other indexers. When the logic errors have been discovered and
fixed, we will re-enable the feature.
  • Loading branch information
darunrs authored Jul 29, 2024
1 parent a46983f commit 674490a
Showing 1 changed file with 34 additions and 30 deletions.
64 changes: 34 additions & 30 deletions coordinator/src/lifecycle.rs
Original file line number Diff line number Diff line change
Expand Up @@ -199,12 +199,13 @@ impl<'a> LifecycleManager<'a> {
#[tracing::instrument(name = "repairing", skip_all)]
async fn handle_repairing(
&self,
config: Option<&IndexerConfig>,
_config: Option<&IndexerConfig>,
_state: &IndexerState,
) -> LifecycleState {
if config.is_none() {
return LifecycleState::Deleting;
}
// TODO: Re-enable auto deprovision once guard rails in place
// if config.is_none() {
// return LifecycleState::Deleting;
// }

// TODO Add more robust error handling, for now just stop
LifecycleState::Repairing
Expand All @@ -228,33 +229,36 @@ impl<'a> LifecycleManager<'a> {
warn!(?error, "Failed to stop executor");
}

if self.state_manager.delete_state(state).await.is_err() {
// Retry
return LifecycleState::Deleting;
}

info!("Clearing block stream");

if self
.redis_client
.del(state.get_redis_stream_key())
.await
.is_err()
{
// Retry
return LifecycleState::Deleting;
}

if self
.data_layer_handler
.ensure_deprovisioned(state.account_id.clone(), state.function_name.clone())
.await
.is_err()
{
return LifecycleState::Deleted;
}
tracing::error!("Temporarily preventing indexer deprovision due to service instability");
LifecycleState::Repairing

LifecycleState::Deleted
// if self.state_manager.delete_state(state).await.is_err() {
// // Retry
// return LifecycleState::Deleting;
// }
//
// info!("Clearing block stream");
//
// if self
// .redis_client
// .del(state.get_redis_stream_key())
// .await
// .is_err()
// {
// // Retry
// return LifecycleState::Deleting;
// }
//
// if self
// .data_layer_handler
// .ensure_deprovisioned(state.account_id.clone(), state.function_name.clone())
// .await
// .is_err()
// {
// return LifecycleState::Deleted;
// }
//
// LifecycleState::Deleted
}

#[tracing::instrument(
Expand Down

0 comments on commit 674490a

Please sign in to comment.