From 102b98f6063efd700f8fc8c7334e4ffe050391a5 Mon Sep 17 00:00:00 2001 From: Heiko Seeberger Date: Tue, 23 May 2023 08:50:43 +0200 Subject: [PATCH 1/3] feat(deployer): add more deployment metadata --- cargo-shuttle/src/client.rs | 12 +-- cargo-shuttle/src/lib.rs | 119 +++++++++++++------------ common/src/models/deployment.rs | 15 ++++ common/src/project.rs | 3 +- deployer/migrations/0003_git.sql | 11 +++ deployer/src/handlers/mod.rs | 40 ++++----- deployer/src/persistence/deployment.rs | 11 ++- deployer/src/persistence/mod.rs | 62 +++++++++---- 8 files changed, 171 insertions(+), 102 deletions(-) create mode 100644 deployer/migrations/0003_git.sql diff --git a/cargo-shuttle/src/client.rs b/cargo-shuttle/src/client.rs index d31d7e85b..58cfef156 100644 --- a/cargo-shuttle/src/client.rs +++ b/cargo-shuttle/src/client.rs @@ -7,6 +7,7 @@ use reqwest_middleware::{ClientBuilder, ClientWithMiddleware, RequestBuilder}; use reqwest_retry::policies::ExponentialBackoff; use reqwest_retry::RetryTransientMiddleware; use serde::{Deserialize, Serialize}; +use shuttle_common::models::deployment::DeploymentRequest; use shuttle_common::models::{deployment, project, secret, service, ToJson}; use shuttle_common::project::ProjectName; use shuttle_common::{resource, ApiKey, ApiUrl, LogItem}; @@ -36,20 +37,15 @@ impl Client { pub async fn deploy( &self, - data: Vec, project: &ProjectName, - no_test: bool, + body: DeploymentRequest, ) -> Result { - let mut path = format!( + let path = format!( "/projects/{}/services/{}", project.as_str(), project.as_str() ); - if no_test { - let _ = write!(path, "?no-test"); - } - let url = format!("{}{}", self.api_url, path); let mut builder = Self::get_retry_client().post(url); @@ -57,7 +53,7 @@ impl Client { builder = self.set_builder_auth(builder); builder - .body(data) + .json(&body) .header("Transfer-Encoding", "chunked") .send() .await diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs index 58389cd53..779f652c4 100644 --- a/cargo-shuttle/src/lib.rs +++ b/cargo-shuttle/src/lib.rs @@ -7,7 +7,9 @@ mod provisioner_server; use args::LogoutArgs; use indicatif::ProgressBar; use shuttle_common::claims::{ClaimService, InjectPropagation}; -use shuttle_common::models::deployment::get_deployments_table; +use shuttle_common::models::deployment::{ + get_deployments_table, DeploymentRequest, GIT_STRINGS_MAX_LENGTH, +}; use shuttle_common::models::project::IDLE_MINUTES; use shuttle_common::models::resource::get_resources_table; use shuttle_common::project::ProjectName; @@ -908,15 +910,48 @@ impl Shuttle { } async fn deploy(&self, client: &Client, args: DeployArgs) -> Result { - if !args.allow_dirty { - self.is_dirty()?; + let working_directory = self.ctx.working_directory(); + + let mut deploy_req: DeploymentRequest = DeploymentRequest { + no_test: args.no_test, + ..Default::default() + }; + + if let Ok(repo) = Repository::discover(working_directory) { + let repo_path = repo + .workdir() + .context("getting working directory of repository")?; + let repo_path = dunce::canonicalize(repo_path)?; + trace!(?repo_path, "found git repository"); + + let is_dirty = self.is_dirty(&repo).err(); + if !args.allow_dirty { + if let Some(err) = is_dirty { + bail!(err); + } + } + deploy_req.git_dirty = Some(is_dirty.is_some()); + + if let Ok(head) = repo.head() { + // This is typically the name of the current branch + // It is "HEAD" when head detached, for example when a tag is checked out + deploy_req.git_branch = head + .shorthand() + .map(|s| s.chars().take(GIT_STRINGS_MAX_LENGTH).collect()); + if let Ok(commit) = head.peel_to_commit() { + deploy_req.git_commit_id = Some(commit.id().to_string()); + // Summary is None if error or invalid utf-8 + deploy_req.git_commit_msg = commit + .summary() + .map(|s| s.chars().take(GIT_STRINGS_MAX_LENGTH).collect()); + } + } } let data = self.make_archive()?; + deploy_req.archive = todo!("Base64 encode the Vec"); - let deployment = client - .deploy(data, self.ctx.project_name(), args.no_test) - .await?; + let deployment = client.deploy(self.ctx.project_name(), deploy_req).await?; let mut stream = client .get_logs_ws(self.ctx.project_name(), &deployment.id) @@ -1197,61 +1232,35 @@ impl Shuttle { Ok(bytes) } - fn is_dirty(&self) -> Result<()> { - let working_directory = self.ctx.working_directory(); - if let Ok(repo) = Repository::discover(working_directory) { - let repo_path = repo - .workdir() - .context("getting working directory of repository")?; - - let repo_path = dunce::canonicalize(repo_path)?; - - trace!(?repo_path, "found git repository"); - - let repo_rel_path = working_directory - .strip_prefix(repo_path.as_path()) - .context("stripping repository path from working directory")?; - - trace!( - ?repo_rel_path, - "got working directory path relative to git repository" + fn is_dirty(&self, repo: &Repository) -> Result<()> { + let mut status_options = StatusOptions::new(); + status_options.include_untracked(true); + let statuses = repo + .statuses(Some(&mut status_options)) + .context("getting status of repository files")?; + + if !statuses.is_empty() { + let mut error = format!( + "{} files in the working directory contain changes that were not yet committed into git:\n", + statuses.len() ); - let mut status_options = StatusOptions::new(); - status_options - .pathspec(repo_rel_path) - .include_untracked(true); - - let statuses = repo - .statuses(Some(&mut status_options)) - .context("getting status of repository files")?; - - if !statuses.is_empty() { - let mut error: String = format!("{} files in the working directory contain changes that were not yet committed into git:", statuses.len()); - writeln!(error).expect("to append error"); - - for status in statuses.iter() { - trace!( - path = status.path(), - status = ?status.status(), - "found file with updates" - ); + for status in statuses.iter() { + trace!( + path = status.path(), + status = ?status.status(), + "found file with updates" + ); - let path = - repo_path.join(status.path().context("getting path of changed file")?); - let rel_path = path - .strip_prefix(working_directory) - .expect("getting relative path of changed file") - .display(); + let rel_path = status.path().context("getting path of changed file")?; - writeln!(error, "{rel_path}").expect("to append error"); - } + writeln!(error, "{rel_path}").expect("to append error"); + } - writeln!(error).expect("to append error"); - writeln!(error, "To proceed despite this and include the uncommitted changes, pass the `--allow-dirty` flag").expect("to append error"); + writeln!(error).expect("to append error"); + writeln!(error, "To proceed despite this and include the uncommitted changes, pass the `--allow-dirty` flag").expect("to append error"); - bail!(error); - } + bail!(error); } Ok(()) diff --git a/common/src/models/deployment.rs b/common/src/models/deployment.rs index 2b4bc00c5..6b6355d93 100644 --- a/common/src/models/deployment.rs +++ b/common/src/models/deployment.rs @@ -115,3 +115,18 @@ Most recent {} for {} ) } } + +#[derive(Default, Deserialize, Serialize)] +#[cfg_attr(feature = "openapi", derive(ToSchema))] +#[cfg_attr(feature = "openapi", schema(as = shuttle_common::models::deployment::DeploymentRequest))] +pub struct DeploymentRequest { + /// Base64 encoded TAR archive + pub archive: String, + pub no_test: bool, + pub git_commit_id: Option, + pub git_commit_msg: Option, + pub git_branch: Option, + pub git_dirty: Option, +} + +pub const GIT_STRINGS_MAX_LENGTH: usize = 80; diff --git a/common/src/project.rs b/common/src/project.rs index 5846de6f8..0b209cdca 100644 --- a/common/src/project.rs +++ b/common/src/project.rs @@ -11,11 +11,10 @@ use std::str::FromStr; /// as per [IETF RFC 1123](https://datatracker.ietf.org/doc/html/rfc1123). /// Initially we'll implement a strict subset of the IETF RFC 1123, concretely: /// - It does not start or end with `-` or `_`. -/// - It does not contain any characters outside of the alphanumeric range, except for `-` or '_'. +/// - It does not contain any characters outside of the alphanumeric range, except for `-` or `_`. /// - It is not empty. /// - It does not contain profanity. /// - It is not a reserved word. -/// #[derive(Clone, Serialize, Debug, Eq, PartialEq)] pub struct ProjectName(String); diff --git a/deployer/migrations/0003_git.sql b/deployer/migrations/0003_git.sql new file mode 100644 index 000000000..d171a4f9b --- /dev/null +++ b/deployer/migrations/0003_git.sql @@ -0,0 +1,11 @@ +ALTER TABLE deployments +ADD COLUMN git_commit_id TEXT; + +ALTER TABLE deployments +ADD COLUMN git_commit_msg TEXT; + +ALTER TABLE deployments +ADD COLUMN git_branch TEXT; + +ALTER TABLE deployments +ADD COLUMN git_dirty BOOLEAN; \ No newline at end of file diff --git a/deployer/src/handlers/mod.rs b/deployer/src/handlers/mod.rs index f7ff0f126..df6b51b9a 100644 --- a/deployer/src/handlers/mod.rs +++ b/deployer/src/handlers/mod.rs @@ -1,16 +1,16 @@ mod error; +use crate::deployment::{DeploymentManager, Queued}; +use crate::persistence::{Deployment, Log, Persistence, ResourceManager, SecretGetter, State}; use axum::extract::ws::{self, WebSocket}; use axum::extract::{Extension, Path, Query}; use axum::handler::Handler; use axum::headers::HeaderMapExt; use axum::middleware::{self, from_extractor}; use axum::routing::{get, post, Router}; -use axum::{extract::BodyStream, Json}; -use bytes::BufMut; +use axum::Json; use chrono::{TimeZone, Utc}; use fqdn::FQDN; -use futures::StreamExt; use hyper::Uri; use serde::Deserialize; use shuttle_common::backends::auth::{ @@ -19,22 +19,17 @@ use shuttle_common::backends::auth::{ use shuttle_common::backends::headers::XShuttleAccountName; use shuttle_common::backends::metrics::{Metrics, TraceLayer}; use shuttle_common::claims::{Claim, Scope}; +use shuttle_common::models::deployment::{DeploymentRequest, GIT_STRINGS_MAX_LENGTH}; use shuttle_common::models::secret; use shuttle_common::project::ProjectName; use shuttle_common::storage_manager::StorageManager; use shuttle_common::{request_span, LogItem}; use shuttle_service::builder::clean_crate; -use tracing::{debug, error, field, instrument, trace, warn}; +use tracing::{error, field, instrument, trace, warn}; use utoipa::{IntoParams, OpenApi}; - use utoipa_swagger_ui::SwaggerUi; use uuid::Uuid; -use crate::deployment::{DeploymentManager, Queued}; -use crate::persistence::{Deployment, Log, Persistence, ResourceManager, SecretGetter, State}; - -use std::collections::HashMap; - pub use {self::error::Error, self::error::Result, self::local::set_jwt_bearer}; mod local; @@ -315,12 +310,13 @@ pub async fn create_service( Extension(deployment_manager): Extension, Extension(claim): Extension, Path((project_name, service_name)): Path<(String, String)>, - Query(params): Query>, - mut stream: BodyStream, + Json(body): Json, ) -> Result> { let service = persistence.get_or_create_service(&service_name).await?; let id = Uuid::new_v4(); + let data = todo!("Base64 decode body.archive"); + let deployment = Deployment { id, service_id: service.id, @@ -328,16 +324,18 @@ pub async fn create_service( last_update: Utc::now(), address: None, is_next: false, + git_commit_id: body + .git_commit_id + .map(|s| s.chars().take(GIT_STRINGS_MAX_LENGTH).collect()), + git_commit_msg: body + .git_commit_msg + .map(|s| s.chars().take(GIT_STRINGS_MAX_LENGTH).collect()), + git_branch: body + .git_branch + .map(|s| s.chars().take(GIT_STRINGS_MAX_LENGTH).collect()), + git_dirty: body.git_dirty, }; - let mut data = Vec::new(); - while let Some(buf) = stream.next().await { - let buf = buf?; - debug!("Received {} bytes", buf.len()); - data.put(buf); - } - debug!("Received a total of {} bytes", data.len()); - persistence.insert_deployment(deployment.clone()).await?; let queued = Queued { @@ -345,7 +343,7 @@ pub async fn create_service( service_name: service.name, service_id: service.id, data, - will_run_tests: !params.contains_key("no-test"), + will_run_tests: !body.no_test, tracing_context: Default::default(), claim: Some(claim), }; diff --git a/deployer/src/persistence/deployment.rs b/deployer/src/persistence/deployment.rs index c79a0ee23..8a7b51841 100644 --- a/deployer/src/persistence/deployment.rs +++ b/deployer/src/persistence/deployment.rs @@ -9,7 +9,8 @@ use uuid::Uuid; use super::state::State; -#[derive(Clone, Debug, Eq, PartialEq, ToSchema)] +// We are using `Option` for the additional `git_*` fields for backward compat. +#[derive(Clone, Debug, Default, Eq, PartialEq, ToSchema)] pub struct Deployment { pub id: Uuid, pub service_id: Uuid, @@ -17,6 +18,10 @@ pub struct Deployment { pub last_update: DateTime, pub address: Option, pub is_next: bool, + pub git_commit_id: Option, + pub git_commit_msg: Option, + pub git_branch: Option, + pub git_dirty: Option, } impl FromRow<'_, SqliteRow> for Deployment { @@ -40,6 +45,10 @@ impl FromRow<'_, SqliteRow> for Deployment { last_update: row.try_get("last_update")?, address, is_next: row.try_get("is_next")?, + git_commit_id: row.try_get("git_commit_id")?, + git_commit_msg: row.try_get("git_commit_msg")?, + git_branch: row.try_get("git_branch")?, + git_dirty: row.try_get("git_dirty")?, }) } } diff --git a/deployer/src/persistence/mod.rs b/deployer/src/persistence/mod.rs index 449fcd106..1345f16c7 100644 --- a/deployer/src/persistence/mod.rs +++ b/deployer/src/persistence/mod.rs @@ -171,19 +171,21 @@ impl Persistence { pub async fn insert_deployment(&self, deployment: impl Into) -> Result<()> { let deployment = deployment.into(); - sqlx::query( - "INSERT INTO deployments (id, service_id, state, last_update, address, is_next) VALUES (?, ?, ?, ?, ?, ?)", - ) - .bind(deployment.id) - .bind(deployment.service_id) - .bind(deployment.state) - .bind(deployment.last_update) - .bind(deployment.address.map(|socket| socket.to_string())) - .bind(deployment.is_next) - .execute(&self.pool) - .await - .map(|_| ()) - .map_err(Error::from) + sqlx::query("INSERT INTO deployments VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)") + .bind(deployment.id) + .bind(deployment.service_id) + .bind(deployment.state) + .bind(deployment.last_update) + .bind(deployment.address.map(|socket| socket.to_string())) + .bind(deployment.is_next) + .bind(deployment.git_commit_id) + .bind(deployment.git_commit_msg) + .bind(deployment.git_branch) + .bind(deployment.git_dirty) + .execute(&self.pool) + .await + .map(|_| ()) + .map_err(Error::from) } pub async fn get_deployment(&self, id: &Uuid) -> Result> { @@ -534,8 +536,7 @@ mod tests { service_id, state: State::Queued, last_update: Utc.with_ymd_and_hms(2022, 4, 25, 4, 43, 33).unwrap(), - address: None, - is_next: false, + ..Default::default() }; let address = SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 12345); @@ -579,6 +580,10 @@ mod tests { last_update: Utc::now(), address: None, is_next: false, + git_commit_id: None, + git_commit_msg: None, + git_branch: None, + git_dirty: None, }) .collect(); @@ -613,6 +618,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2022, 4, 25, 7, 29, 35).unwrap(), address: None, is_next: false, + ..Default::default() }; let deployment_stopped = Deployment { id: Uuid::new_v4(), @@ -621,6 +627,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2022, 4, 25, 7, 49, 35).unwrap(), address: None, is_next: false, + ..Default::default() }; let deployment_other = Deployment { id: Uuid::new_v4(), @@ -629,6 +636,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2022, 4, 25, 7, 39, 39).unwrap(), address: None, is_next: false, + ..Default::default() }; let deployment_running = Deployment { id: Uuid::new_v4(), @@ -637,6 +645,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2022, 4, 25, 7, 48, 29).unwrap(), address: Some(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 9876)), is_next: true, + ..Default::default() }; for deployment in [ @@ -668,6 +677,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2023, 4, 17, 1, 1, 2).unwrap(), address: None, is_next: false, + ..Default::default() }; let deployment_crashed = Deployment { id: Uuid::new_v4(), @@ -676,6 +686,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2023, 4, 17, 1, 1, 2).unwrap(), // second address: None, is_next: false, + ..Default::default() }; let deployment_stopped = Deployment { id: Uuid::new_v4(), @@ -684,6 +695,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2023, 4, 17, 1, 1, 1).unwrap(), // first address: None, is_next: false, + ..Default::default() }; let deployment_running = Deployment { id: Uuid::new_v4(), @@ -692,6 +704,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2023, 4, 17, 1, 1, 3).unwrap(), // third address: Some(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 9876)), is_next: true, + ..Default::default() }; for deployment in [ @@ -728,6 +741,7 @@ mod tests { last_update: time, address: None, is_next: false, + ..Default::default() }; let deployment_stopped = Deployment { id: Uuid::new_v4(), @@ -736,6 +750,7 @@ mod tests { last_update: time.checked_add_signed(Duration::seconds(1)).unwrap(), address: None, is_next: false, + ..Default::default() }; let deployment_running = Deployment { id: Uuid::new_v4(), @@ -744,6 +759,7 @@ mod tests { last_update: time.checked_add_signed(Duration::seconds(2)).unwrap(), address: Some(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 9876)), is_next: false, + ..Default::default() }; let deployment_queued = Deployment { id: Uuid::new_v4(), @@ -752,6 +768,7 @@ mod tests { last_update: time.checked_add_signed(Duration::seconds(3)).unwrap(), address: None, is_next: false, + ..Default::default() }; let deployment_building = Deployment { id: Uuid::new_v4(), @@ -760,6 +777,7 @@ mod tests { last_update: time.checked_add_signed(Duration::seconds(4)).unwrap(), address: None, is_next: false, + ..Default::default() }; let deployment_built = Deployment { id: Uuid::new_v4(), @@ -768,6 +786,7 @@ mod tests { last_update: time.checked_add_signed(Duration::seconds(5)).unwrap(), address: None, is_next: true, + ..Default::default() }; let deployment_loading = Deployment { id: Uuid::new_v4(), @@ -776,6 +795,7 @@ mod tests { last_update: time.checked_add_signed(Duration::seconds(6)).unwrap(), address: None, is_next: false, + ..Default::default() }; for deployment in [ @@ -835,6 +855,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2022, 4, 25, 4, 29, 33).unwrap(), address: None, is_next: false, + ..Default::default() }, Deployment { id: id_1, @@ -843,6 +864,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2022, 4, 25, 4, 29, 44).unwrap(), address: None, is_next: false, + ..Default::default() }, Deployment { id: id_2, @@ -851,6 +873,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2022, 4, 25, 4, 33, 48).unwrap(), address: None, is_next: true, + ..Default::default() }, Deployment { id: Uuid::new_v4(), @@ -859,6 +882,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2022, 4, 25, 4, 38, 52).unwrap(), address: None, is_next: true, + ..Default::default() }, Deployment { id: id_3, @@ -867,6 +891,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2022, 4, 25, 4, 42, 32).unwrap(), address: None, is_next: false, + ..Default::default() }, ] { p.insert_deployment(deployment).await.unwrap(); @@ -1019,6 +1044,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2022, 4, 29, 2, 39, 39).unwrap(), address: None, is_next: false, + ..Default::default() }) .await .unwrap(); @@ -1059,6 +1085,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2022, 4, 29, 2, 39, 59).unwrap(), address: None, is_next: false, + ..Default::default() } ); } @@ -1240,6 +1267,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2022, 4, 25, 4, 29, 33).unwrap(), address: None, is_next: false, + ..Default::default() }, Deployment { id: Uuid::new_v4(), @@ -1248,6 +1276,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2022, 4, 25, 4, 29, 44).unwrap(), address: None, is_next: false, + ..Default::default() }, Deployment { id: id_1, @@ -1256,6 +1285,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2022, 4, 25, 4, 33, 48).unwrap(), address: None, is_next: false, + ..Default::default() }, Deployment { id: Uuid::new_v4(), @@ -1264,6 +1294,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2022, 4, 25, 4, 38, 52).unwrap(), address: None, is_next: false, + ..Default::default() }, Deployment { id: id_2, @@ -1272,6 +1303,7 @@ mod tests { last_update: Utc.with_ymd_and_hms(2022, 4, 25, 4, 42, 32).unwrap(), address: None, is_next: true, + ..Default::default() }, ] { p.insert_deployment(deployment).await.unwrap(); From 8aa81442a5548e9fc007c765eeb09341ce7aa9b4 Mon Sep 17 00:00:00 2001 From: Heiko Seeberger Date: Tue, 6 Jun 2023 15:39:42 +0200 Subject: [PATCH 2/3] rmp_serde --- Cargo.lock | 2 ++ cargo-shuttle/Cargo.toml | 1 + cargo-shuttle/src/client.rs | 10 +++---- cargo-shuttle/src/lib.rs | 17 +++++------ common/src/models/deployment.rs | 10 ++----- deployer/Cargo.toml | 1 + deployer/src/handlers/mod.rs | 51 +++++++++++++++++++++++++-------- 7 files changed, 59 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ff018184a..7a4202e0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1102,6 +1102,7 @@ dependencies = [ "reqwest-middleware", "reqwest-retry", "rexpect", + "rmp-serde", "semver 1.0.17", "serde", "serde_json", @@ -4810,6 +4811,7 @@ dependencies = [ "pipe", "portpicker", "rand", + "rmp-serde", "serde", "serde_json", "shuttle-common", diff --git a/cargo-shuttle/Cargo.toml b/cargo-shuttle/Cargo.toml index 0c937311b..716dfa780 100644 --- a/cargo-shuttle/Cargo.toml +++ b/cargo-shuttle/Cargo.toml @@ -36,6 +36,7 @@ portpicker = { workspace = true } reqwest = { workspace = true, features = ["json"] } reqwest-middleware = "0.2.0" reqwest-retry = "0.2.0" +rmp-serde = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } sqlx = { workspace = true, features = ["runtime-tokio-native-tls", "postgres"] } diff --git a/cargo-shuttle/src/client.rs b/cargo-shuttle/src/client.rs index 58cfef156..d3ef77631 100644 --- a/cargo-shuttle/src/client.rs +++ b/cargo-shuttle/src/client.rs @@ -1,5 +1,3 @@ -use std::fmt::Write; - use anyhow::{Context, Result}; use headers::{Authorization, HeaderMapExt}; use reqwest::Response; @@ -38,23 +36,23 @@ impl Client { pub async fn deploy( &self, project: &ProjectName, - body: DeploymentRequest, + deployment_req: DeploymentRequest, ) -> Result { let path = format!( "/projects/{}/services/{}", project.as_str(), project.as_str() ); + let deployment_req = rmp_serde::to_vec(&deployment_req) + .context("serialize DeploymentRequest as a MessagePack byte vector")?; let url = format!("{}{}", self.api_url, path); - let mut builder = Self::get_retry_client().post(url); - builder = self.set_builder_auth(builder); builder - .json(&body) .header("Transfer-Encoding", "chunked") + .body(deployment_req) .send() .await .context("failed to send deployment to the Shuttle server")? diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs index 779f652c4..f9384b51e 100644 --- a/cargo-shuttle/src/lib.rs +++ b/cargo-shuttle/src/lib.rs @@ -912,7 +912,7 @@ impl Shuttle { async fn deploy(&self, client: &Client, args: DeployArgs) -> Result { let working_directory = self.ctx.working_directory(); - let mut deploy_req: DeploymentRequest = DeploymentRequest { + let mut deployment_req: DeploymentRequest = DeploymentRequest { no_test: args.no_test, ..Default::default() }; @@ -930,28 +930,29 @@ impl Shuttle { bail!(err); } } - deploy_req.git_dirty = Some(is_dirty.is_some()); + deployment_req.git_dirty = Some(is_dirty.is_some()); if let Ok(head) = repo.head() { // This is typically the name of the current branch // It is "HEAD" when head detached, for example when a tag is checked out - deploy_req.git_branch = head + deployment_req.git_branch = head .shorthand() .map(|s| s.chars().take(GIT_STRINGS_MAX_LENGTH).collect()); if let Ok(commit) = head.peel_to_commit() { - deploy_req.git_commit_id = Some(commit.id().to_string()); + deployment_req.git_commit_id = Some(commit.id().to_string()); // Summary is None if error or invalid utf-8 - deploy_req.git_commit_msg = commit + deployment_req.git_commit_msg = commit .summary() .map(|s| s.chars().take(GIT_STRINGS_MAX_LENGTH).collect()); } } } - let data = self.make_archive()?; - deploy_req.archive = todo!("Base64 encode the Vec"); + deployment_req.data = self.make_archive()?; - let deployment = client.deploy(self.ctx.project_name(), deploy_req).await?; + let deployment = client + .deploy(self.ctx.project_name(), deployment_req) + .await?; let mut stream = client .get_logs_ws(self.ctx.project_name(), &deployment.id) diff --git a/common/src/models/deployment.rs b/common/src/models/deployment.rs index 6b6355d93..8b57910ce 100644 --- a/common/src/models/deployment.rs +++ b/common/src/models/deployment.rs @@ -1,5 +1,4 @@ -use std::{fmt::Display, str::FromStr}; - +use crate::deployment::State; use chrono::{DateTime, Utc}; use comfy_table::{ modifiers::UTF8_ROUND_CORNERS, presets::UTF8_FULL, Attribute, Cell, CellAlignment, Color, @@ -7,13 +6,11 @@ use comfy_table::{ }; use crossterm::style::Stylize; use serde::{Deserialize, Serialize}; - +use std::{fmt::Display, str::FromStr}; #[cfg(feature = "openapi")] use utoipa::ToSchema; use uuid::Uuid; -use crate::deployment::State; - #[derive(Deserialize, Serialize)] #[cfg_attr(feature = "openapi", derive(ToSchema))] #[cfg_attr(feature = "openapi", schema(as = shuttle_common::models::deployment::Response))] @@ -120,8 +117,7 @@ Most recent {} for {} #[cfg_attr(feature = "openapi", derive(ToSchema))] #[cfg_attr(feature = "openapi", schema(as = shuttle_common::models::deployment::DeploymentRequest))] pub struct DeploymentRequest { - /// Base64 encoded TAR archive - pub archive: String, + pub data: Vec, pub no_test: bool, pub git_commit_id: Option, pub git_commit_msg: Option, diff --git a/deployer/Cargo.toml b/deployer/Cargo.toml index 4e5e58560..65adc5ea5 100644 --- a/deployer/Cargo.toml +++ b/deployer/Cargo.toml @@ -25,6 +25,7 @@ opentelemetry = { workspace = true } opentelemetry-http = { workspace = true } pipe = { workspace = true } portpicker = { workspace = true } +rmp-serde = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sqlx = { workspace = true, features = [ diff --git a/deployer/src/handlers/mod.rs b/deployer/src/handlers/mod.rs index df6b51b9a..637dcaae1 100644 --- a/deployer/src/handlers/mod.rs +++ b/deployer/src/handlers/mod.rs @@ -2,17 +2,22 @@ mod error; use crate::deployment::{DeploymentManager, Queued}; use crate::persistence::{Deployment, Log, Persistence, ResourceManager, SecretGetter, State}; -use axum::extract::ws::{self, WebSocket}; +use async_trait::async_trait; +use axum::extract::{ + ws::{self, WebSocket}, + FromRequest, +}; use axum::extract::{Extension, Path, Query}; use axum::handler::Handler; use axum::headers::HeaderMapExt; use axum::middleware::{self, from_extractor}; use axum::routing::{get, post, Router}; use axum::Json; +use bytes::Bytes; use chrono::{TimeZone, Utc}; use fqdn::FQDN; -use hyper::Uri; -use serde::Deserialize; +use hyper::{Request, StatusCode, Uri}; +use serde::{de::DeserializeOwned, Deserialize}; use shuttle_common::backends::auth::{ AdminSecretLayer, AuthPublicKey, JwtAuthenticationLayer, ScopedLayer, }; @@ -310,13 +315,11 @@ pub async fn create_service( Extension(deployment_manager): Extension, Extension(claim): Extension, Path((project_name, service_name)): Path<(String, String)>, - Json(body): Json, + Rmp(deployment_req): Rmp, ) -> Result> { let service = persistence.get_or_create_service(&service_name).await?; let id = Uuid::new_v4(); - let data = todo!("Base64 decode body.archive"); - let deployment = Deployment { id, service_id: service.id, @@ -324,16 +327,16 @@ pub async fn create_service( last_update: Utc::now(), address: None, is_next: false, - git_commit_id: body + git_commit_id: deployment_req .git_commit_id .map(|s| s.chars().take(GIT_STRINGS_MAX_LENGTH).collect()), - git_commit_msg: body + git_commit_msg: deployment_req .git_commit_msg .map(|s| s.chars().take(GIT_STRINGS_MAX_LENGTH).collect()), - git_branch: body + git_branch: deployment_req .git_branch .map(|s| s.chars().take(GIT_STRINGS_MAX_LENGTH).collect()), - git_dirty: body.git_dirty, + git_dirty: deployment_req.git_dirty, }; persistence.insert_deployment(deployment.clone()).await?; @@ -342,8 +345,8 @@ pub async fn create_service( id, service_name: service.name, service_id: service.id, - data, - will_run_tests: !body.no_test, + data: deployment_req.data, + will_run_tests: !deployment_req.no_test, tracing_context: Default::default(), claim: Some(claim), }; @@ -647,3 +650,27 @@ pub async fn clean_project( async fn get_status() -> String { "Ok".to_string() } + +pub struct Rmp(T); + +#[async_trait] +impl FromRequest for Rmp +where + S: Send + Sync, + B: Send + 'static, + Bytes: FromRequest, + T: DeserializeOwned, +{ + type Rejection = StatusCode; + + async fn from_request( + req: Request, + state: &S, + ) -> std::result::Result { + let bytes = Bytes::from_request(req, state) + .await + .map_err(|_| StatusCode::BAD_REQUEST)?; + let t = rmp_serde::from_slice::(&bytes).map_err(|_| StatusCode::BAD_REQUEST)?; + Ok(Self(t)) + } +} From 19ff27c8f2b5cbaa2bf59e5377d5ac7d131c1601 Mon Sep 17 00:00:00 2001 From: Heiko Seeberger Date: Wed, 7 Jun 2023 13:23:23 +0200 Subject: [PATCH 3/3] review comments --- cargo-shuttle/src/lib.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs index f9384b51e..72ef8e2c3 100644 --- a/cargo-shuttle/src/lib.rs +++ b/cargo-shuttle/src/lib.rs @@ -924,13 +924,10 @@ impl Shuttle { let repo_path = dunce::canonicalize(repo_path)?; trace!(?repo_path, "found git repository"); - let is_dirty = self.is_dirty(&repo).err(); if !args.allow_dirty { - if let Some(err) = is_dirty { - bail!(err); - } + self.is_dirty(&repo).context("dirty not allowed")?; } - deployment_req.git_dirty = Some(is_dirty.is_some()); + deployment_req.git_dirty = Some(self.is_dirty(&repo).is_err()); if let Ok(head) = repo.head() { // This is typically the name of the current branch