diff --git a/common/src/models/error.rs b/common/src/models/error.rs index 52e5ae00d..46e5c29f7 100644 --- a/common/src/models/error.rs +++ b/common/src/models/error.rs @@ -61,7 +61,7 @@ pub enum ErrorKind { NotReady, ServiceUnavailable, DeleteProjectFailed, - ContainerLimit, + CapacityLimit, } impl From for ApiError { @@ -132,7 +132,7 @@ impl From for ApiError { ErrorKind::Forbidden => (StatusCode::FORBIDDEN, "Forbidden"), ErrorKind::NotReady => (StatusCode::INTERNAL_SERVER_ERROR, "Service not ready"), ErrorKind::DeleteProjectFailed => (StatusCode::INTERNAL_SERVER_ERROR, "Deleting project failed"), - ErrorKind::ContainerLimit => (StatusCode::SERVICE_UNAVAILABLE, "Our server is full and cannot create / start projects at this time"), + ErrorKind::CapacityLimit => (StatusCode::SERVICE_UNAVAILABLE, "Our server is at capacity and cannot serve your request at this time"), }; Self { message: error_message.to_string(), diff --git a/gateway/src/api/latest.rs b/gateway/src/api/latest.rs index 70cda5e0e..03776aae1 100644 --- a/gateway/src/api/latest.rs +++ b/gateway/src/api/latest.rs @@ -219,7 +219,9 @@ async fn create_project( .saturating_sub(is_cch_project as u32), ); - service.has_capacity(is_cch_project, &claim.tier).await?; + if !claim.is_admin() { + service.has_capacity(is_cch_project, &claim.tier).await?; + } let project = service .create_project( @@ -442,9 +444,11 @@ async fn route_project( let project_name = scoped_user.scope; let is_cch_project = project_name.is_cch_project(); - service - .has_capacity(is_cch_project, &scoped_user.user.claim.tier) - .await?; + if !scoped_user.user.claim.is_admin() { + service + .has_capacity(is_cch_project, &scoped_user.user.claim.tier) + .await?; + } let project = service.find_or_start_project(&project_name, sender).await?; service diff --git a/gateway/src/service.rs b/gateway/src/service.rs index b50eda557..6521ae3f0 100644 --- a/gateway/src/service.rs +++ b/gateway/src/service.rs @@ -960,7 +960,13 @@ impl GatewayService { ) -> Result<(), Error> { let current_container_count = self.count_ready_projects().await?; - let has_capacity = if current_container_count < self.cch_container_limit { + let has_capacity = if is_cch_project + && std::fs::metadata("/var/lib/shuttle/BLOCK_CCH23_PROJECT_TRAFFIC").is_ok() + { + // If this control file exists, block routing to cch23 projects. + // Used for emergency load mitigation + return Err(Error::from_kind(ErrorKind::CapacityLimit)); + } else if current_container_count < self.cch_container_limit { true } else if current_container_count < self.soft_container_limit { !is_cch_project @@ -973,7 +979,7 @@ impl GatewayService { if has_capacity { Ok(()) } else { - Err(Error::from_kind(ErrorKind::ContainerLimit)) + Err(Error::from_kind(ErrorKind::CapacityLimit)) } } }