bug: remove project resources automatically when delete the project

common-tests/Cargo.toml

@@ -8,7 +8,7 @@ publish = false
 
 [dependencies]
 cargo-shuttle = { path = "../cargo-shuttle" }
-shuttle-proto = { workspace = true, features = ["builder", "logger"] }
+shuttle-proto = { workspace = true, features = ["builder", "logger", "resource-recorder"] }
 
 hyper = { workspace = true }
 portpicker = { workspace = true }

common-tests/src/lib.rs

@@ -1,6 +1,7 @@
 pub mod builder;
 pub mod cargo_shuttle;
 pub mod logger;
+pub mod resource_recorder;
 
 use shuttle_common::claims::{AccountTier, Claim, Scope};
 

common-tests/src/resource_recorder.rs

@@ -0,0 +1,162 @@
+use std::{
+    net::{Ipv4Addr, SocketAddr},
+    sync::Mutex,
+};
+
+use portpicker::pick_unused_port;
+use shuttle_proto::resource_recorder::{
+    resource_recorder_server::{ResourceRecorder, ResourceRecorderServer},
+    ProjectResourcesRequest, RecordRequest, Resource, ResourceIds, ResourceResponse,
+    ResourcesResponse, ResultResponse, ServiceResourcesRequest,
+};
+use tonic::{async_trait, transport::Server, Request, Response, Status};
+
+struct MockedResourceRecorder {
+    resources: Mutex<Vec<Resource>>,
+}
+
+#[async_trait]
+impl ResourceRecorder for MockedResourceRecorder {
+    async fn record_resources(
+        &self,
+        request: Request<RecordRequest>,
+    ) -> Result<Response<ResultResponse>, Status> {
+        println!("recording resources");
+
+        let RecordRequest {
+            project_id,
+            service_id,
+            resources,
+        } = request.into_inner();
+
+        let mut resources = resources
+            .into_iter()
+            .map(|r| Resource {
+                project_id: project_id.clone(),
+                service_id: service_id.clone(),
+                r#type: r.r#type,
+                config: r.config,
+                data: r.data,
+                is_active: true,
+                created_at: None,
+                last_updated: None,
+            })
+            .collect();
+
+        self.resources.lock().unwrap().append(&mut resources);
+
+        Ok(Response::new(ResultResponse {
+            success: true,
+            message: Default::default(),
+        }))
+    }
+
+    async fn get_project_resources(
+        &self,
+        _request: Request<ProjectResourcesRequest>,
+    ) -> Result<Response<ResourcesResponse>, Status> {
+        println!("getting project resources");
+        Ok(Response::new(Default::default()))
+    }
+
+    async fn get_service_resources(
+        &self,
+        request: Request<ServiceResourcesRequest>,
+    ) -> Result<Response<ResourcesResponse>, Status> {
+        println!("getting service resources");
+
+        let ServiceResourcesRequest { service_id } = request.into_inner();
+        let resources = self
+            .resources
+            .lock()
+            .unwrap()
+            .iter()
+            .filter(|r| r.service_id == service_id)
+            .cloned()
+            .collect();
+
+        Ok(Response::new(ResourcesResponse {
+            success: true,
+            message: Default::default(),
+            resources,
+        }))
+    }
+
+    async fn get_resource(
+        &self,
+        request: tonic::Request<ResourceIds>,
+    ) -> Result<Response<ResourceResponse>, Status> {
+        println!("getting resource");
+
+        let ResourceIds {
+            project_id,
+            service_id,
+            r#type,
+        } = request.into_inner();
+        let resource = self
+            .resources
+            .lock()
+            .unwrap()
+            .iter()
+            .find(|r| {
+                r.project_id == project_id && r.service_id == service_id && r.r#type == r#type
+            })
+            .cloned();
+
+        Ok(Response::new(ResourceResponse {
+            success: resource.is_some(),
+            message: Default::default(),
+            resource,
+        }))
+    }
+
+    async fn delete_resource(
+        &self,
+        request: tonic::Request<ResourceIds>,
+    ) -> Result<Response<ResultResponse>, Status> {
+        println!("delete resource");
+
+        let ResourceIds {
+            project_id,
+            service_id,
+            r#type,
+        } = request.into_inner();
+
+        // Fail to delete a metadata resource if requested
+        if r#type == "metadata" {
+            return Ok(Response::new(ResultResponse {
+                success: false,
+                message: Default::default(),
+            }));
+        }
+
+        self.resources.lock().unwrap().retain(|r| {
+            !(r.project_id == project_id && r.service_id == service_id && r.r#type == r#type)
+        });
+
+        Ok(Response::new(ResultResponse {
+            success: true,
+            message: Default::default(),
+        }))
+    }
+}
+
+/// Start a mocked resource recorder and return the address it started on
+/// This mock will function like a normal resource recorder. However, it will always fail to delete metadata resources
+/// if any tests need to simulate a failure.
+pub async fn start_mocked_resource_recorder() -> u16 {
+    let resource_recorder = MockedResourceRecorder {
+        resources: Mutex::new(Vec::new()),
+    };
+
+    let port = pick_unused_port().unwrap();
+    let resource_recorder_addr = SocketAddr::new(Ipv4Addr::UNSPECIFIED.into(), port);
+    tokio::spawn(async move {
+        Server::builder()
+            .add_service(ResourceRecorderServer::new(resource_recorder))
+            .serve(resource_recorder_addr)
+            .await
+    });
+
+    port
+}

common/src/models/error.rs

@@ -97,8 +97,8 @@ impl From<ErrorKind> for ApiError {
             ErrorKind::ProjectHasResources(resources) => {
                 let resources = resources.join(", ");
                 return Self {
-                    message: format!("Project has resources: {}. Use `cargo shuttle resource list` and `cargo shuttle resource delete <type>` to delete them.", resources),
-                    status_code: StatusCode::FORBIDDEN.as_u16(),
+                    message: format!("Could not automatically delete the following resources: {}. Reach out to our support and ask them for a helping hand.", resources),
+                    status_code: StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
                 }
             }
             ErrorKind::InvalidProjectName(err) => {

gateway/Cargo.toml

@@ -61,9 +61,13 @@ ulid = { workspace = true, features = ["serde"] }
 [dev-dependencies]
 anyhow = { workspace = true }
 colored = { workspace = true }
+flate2 = { workspace = true }
 jsonwebtoken = { workspace = true }
 portpicker = { workspace = true }
 ring = { workspace = true }
+rmp-serde = { workspace = true }
+shuttle-common-tests = { workspace = true }
 snailquote = "0.3.1"
+tar = { workspace = true }
 tempfile = { workspace = true }
 test-context = "0.1.4"

gateway/src/api/latest.rs

@@ -14,7 +14,7 @@ use axum::routing::{any, delete, get, post};
 use axum::{Json as AxumJson, Router};
 use fqdn::FQDN;
 use futures::Future;
-use http::{StatusCode, Uri};
+use http::{Method, StatusCode, Uri};
 use instant_acme::{AccountCredentials, ChallengeType};
 use serde::{Deserialize, Serialize};
 use shuttle_common::backends::auth::{AuthPublicKey, JwtAuthenticationLayer, ScopedLayer};
@@ -370,7 +370,7 @@ async fn delete_project(
     }
 
     // check if database in resources
-    let mut rb = hyper::Request::builder();
+    let mut rb = Request::builder();
     rb.headers_mut().unwrap().clone_from(req.headers());
     let resource_req = rb
         .uri(
@@ -381,7 +381,7 @@ async fn delete_project(
         .method("GET")
         .body(hyper::Body::empty())
         .unwrap();
-    let res = route_project(State(state.clone()), scoped_user, resource_req).await?;
+    let res = route_project(State(state.clone()), scoped_user.clone(), resource_req).await?;
     // 404 == no service == no resources
     if res.status() != StatusCode::NOT_FOUND {
         if res.status() != StatusCode::OK {
@@ -394,20 +394,33 @@ async fn delete_project(
             serde_json::from_slice(&body_bytes)
                 .map_err(|e| Error::source(ErrorKind::Internal, e))?;
 
-        let resources = resources
-            .into_iter()
-            .filter(|resource| {
-                matches!(
-                    resource.r#type,
-                    shuttle_common::resource::Type::Database(_)
-                        | shuttle_common::resource::Type::Secrets
+        let mut delete_fails = Vec::new();
+
+        for resource in resources {
+            let resource_type = resource.r#type.to_string();
+            let mut rb = Request::builder();
+            rb.headers_mut().unwrap().clone_from(req.headers());
+            let resource_req = rb
+                .uri(
+                    format!("/projects/{project_name}/services/{project_name}/resources/{resource_type}")
+                        .parse::<Uri>()
+                        .unwrap(),
                 )
-            })
-            .map(|resource| resource.r#type.to_string())
-            .collect::<Vec<_>>();
+                .method(Method::DELETE)
+                .body(hyper::Body::empty())
+                .unwrap();
+            let res =
+                route_project(State(state.clone()), scoped_user.clone(), resource_req).await?;
+
+            if res.status() != StatusCode::OK {
+                delete_fails.push(resource_type)
+            }
+        }
 
-        if !resources.is_empty() {
-            return Err(Error::from_kind(ErrorKind::ProjectHasResources(resources)));
+        if !delete_fails.is_empty() {
+            return Err(Error::from_kind(ErrorKind::ProjectHasResources(
+                delete_fails,
+            )));
         }
     }
 
@@ -1387,19 +1400,48 @@ pub mod tests {
 
     #[test_context(TestProject)]
     #[tokio::test]
-    async fn api_delete_project_that_is_ready(project: &mut TestProject) -> anyhow::Result<()> {
-        project.router_call(Method::DELETE, "/delete").await;
-
-        Ok(())
+    async fn api_delete_project_that_is_ready(project: &mut TestProject) {
+        assert_eq!(
+            project.router_call(Method::DELETE, "/delete").await,
+            StatusCode::OK
+        );
     }
 
     #[test_context(TestProject)]
     #[tokio::test]
-    async fn api_delete_project_that_is_destroyed(project: &mut TestProject) -> anyhow::Result<()> {
+    async fn api_delete_project_that_is_destroyed(project: &mut TestProject) {
         project.destroy_project().await;
-        project.router_call(Method::DELETE, "/delete").await;
 
-        Ok(())
+        assert_eq!(
+            project.router_call(Method::DELETE, "/delete").await,
+            StatusCode::OK
+        );
+    }
+
+    #[test_context(TestProject)]
+    #[tokio::test]
+    async fn api_delete_project_that_has_resources(project: &mut TestProject) {
+        project.deploy("../examples/rocket/secrets").await;
+        project.stop_service().await;
+
+        assert_eq!(
+            project.router_call(Method::DELETE, "/delete").await,
+            StatusCode::OK
+        );
+    }
+
+    #[test_context(TestProject)]
+    #[tokio::test]
+    async fn api_delete_project_that_has_resources_but_fails_to_remove_them(
+        project: &mut TestProject,
+    ) {
+        project.deploy("../examples/axum/metadata").await;
+        project.stop_service().await;
+
+        assert_eq!(
+            project.router_call(Method::DELETE, "/delete").await,
+            StatusCode::INTERNAL_SERVER_ERROR
+        );
     }
 
     #[tokio::test(flavor = "multi_thread")]

gateway/src/args.rs

@@ -62,6 +62,9 @@ pub struct ContextArgs {
     /// Address to reach the authentication service at
     #[arg(long, default_value = "http://127.0.0.1:8008")]
     pub auth_uri: Uri,
+    /// Address to reach the resource recorder service at
+    #[arg(long, default_value = "http://resource-recorder:8000")]
+    pub resource_recorder_uri: Uri,
     /// The Docker Network name in which to deploy user runtimes
     #[arg(long, default_value = "shuttle_default")]
     pub network_name: String,