diff --git a/Cargo.lock b/Cargo.lock index 894412d79..c1fa09851 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,17 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "ahash" version = "0.8.11" @@ -930,6 +941,27 @@ dependencies = [ "either", ] +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.11+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc" +dependencies = [ + "cc", + "libc", + "pkg-config", +] + [[package]] name = "camino" version = "1.1.6" @@ -1002,6 +1034,7 @@ dependencies = [ "uuid", "walkdir", "webbrowser", + "zip", ] [[package]] @@ -1042,9 +1075,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.37" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e" +checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1055,6 +1088,16 @@ dependencies = [ "windows-targets 0.52.4", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clap" version = "4.5.4" @@ -1192,6 +1235,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "convert_case" version = "0.4.0" @@ -3236,6 +3285,15 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "64e9829a50b42bb782c1df523f78d332fe371b10c661e78b7a3c34b0198e9fac" +[[package]] +name = "inout" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0c10553d664a4d0bcff9f4215d0aac67a639cc68ef660840afe309b807bc9f5" +dependencies = [ + "generic-array", +] + [[package]] name = "instant" version = "0.1.12" @@ -4033,6 +4091,17 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "paste" version = "1.0.14" @@ -4046,6 +4115,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" dependencies = [ "digest", + "hmac", + "password-hash", + "sha2", ] [[package]] @@ -7445,3 +7517,52 @@ name = "zeroize" version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" + +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "aes", + "byteorder", + "bzip2", + "constant_time_eq", + "crc32fast", + "crossbeam-utils", + "flate2", + "hmac", + "pbkdf2", + "sha1", + "time", + "zstd", +] + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.10+zstd.1.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/cargo-shuttle/Cargo.toml b/cargo-shuttle/Cargo.toml index bd541b49c..b36ee6d89 100644 --- a/cargo-shuttle/Cargo.toml +++ b/cargo-shuttle/Cargo.toml @@ -65,6 +65,7 @@ url = { workspace = true } uuid = { workspace = true, features = ["v4"] } walkdir = "2.3.3" webbrowser = "0.8.2" +zip = "0.6.6" [dev-dependencies] assert_cmd = "2.0.6" diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs index 537f7da3b..e4adddbb7 100644 --- a/cargo-shuttle/src/lib.rs +++ b/cargo-shuttle/src/lib.rs @@ -9,7 +9,7 @@ use std::collections::{BTreeMap, HashMap}; use std::ffi::OsString; use std::fmt::Write as FmtWrite; use std::fs::{read_to_string, File}; -use std::io::stdout; +use std::io::{stdout, Read, Write}; use std::net::{Ipv4Addr, SocketAddr}; use std::path::{Path, PathBuf}; use std::process::exit; @@ -57,6 +57,7 @@ use shuttle_proto::{ provisioner::{provisioner_server::Provisioner, DatabaseRequest}, runtime::{self, LoadRequest, StartRequest, StopRequest}, }; +use shuttle_service::builder::{async_cargo_metadata, find_shuttle_packages}; use shuttle_service::{ builder::{build_workspace, BuiltService}, runner, Environment, @@ -69,6 +70,7 @@ use tokio::time::{sleep, Duration}; use tonic::{Request, Status}; use tracing::{debug, error, trace, warn}; use uuid::Uuid; +use zip::write::FileOptions; pub use crate::args::{Command, ProjectArgs, RunArgs, ShuttleArgs}; use crate::args::{ @@ -1656,11 +1658,26 @@ impl Shuttle { let client = self.client.as_ref().unwrap(); let working_directory = self.ctx.working_directory(); - let mut deployment_req: DeploymentRequest = DeploymentRequest { + let mut deployment_req = DeploymentRequest { no_test: args.no_test, ..Default::default() }; + if self.beta { + let manifest_path = working_directory.join("Cargo.toml"); + if !manifest_path.exists() { + bail!("Cargo manifest file not found: {}", manifest_path.display()); + } + let metadata = async_cargo_metadata(manifest_path.as_path()).await?; + let packages = find_shuttle_packages(&metadata)?; + let package_name = packages + .first() + .expect("at least one shuttle crate in the workspace") + .name + .to_owned(); + deployment_req.package_name = Some(package_name); + } + if let Ok(repo) = Repository::discover(working_directory) { let repo_path = repo .workdir() @@ -1690,7 +1707,7 @@ impl Shuttle { } } - deployment_req.data = self.make_archive(args.secret_args.secrets.clone())?; + deployment_req.data = self.make_archive(args.secret_args.secrets.clone(), self.beta)?; if deployment_req.data.len() > CREATE_SERVICE_BODY_LIMIT { bail!( r#"The project is too large - the limit is {} MB. \ @@ -2134,10 +2151,8 @@ impl Shuttle { Ok(CommandOutcome::Ok) } - fn make_archive(&self, secrets_file: Option) -> Result> { + fn make_archive(&self, secrets_file: Option, zip: bool) -> Result> { let include_patterns = self.ctx.assets(); - let encoder = GzEncoder::new(Vec::new(), Compression::new(3)); - let mut tar = Builder::new(encoder); let working_directory = self.ctx.working_directory(); @@ -2205,8 +2220,14 @@ impl Shuttle { continue; } + // zip file puts all files in root, tar puts all files nested in a dir at root level + let prefix = if zip { + working_directory + } else { + working_directory.parent().context("get parent dir")? + }; let mut name = path - .strip_prefix(working_directory.parent().context("get parent dir")?) + .strip_prefix(prefix) .context("strip prefix of path")? .to_owned(); @@ -2224,14 +2245,34 @@ impl Shuttle { bail!("No files included in upload."); } - // Append all the entries to the archive. - for (k, v) in archive_files { - debug!("Packing {k:?}"); - tar.append_path_with_name(k, v)?; - } + let bytes = if zip { + debug!("making zip archive"); + let mut zip = zip::ZipWriter::new(std::io::Cursor::new(Vec::new())); + for (path, name) in archive_files { + debug!("Packing {path:?}"); + zip.start_file( + name.to_str().expect("valid filename"), + FileOptions::default(), + )?; + let mut b = Vec::new(); + File::open(path)?.read_to_end(&mut b)?; + zip.write_all(&b)?; + } + let r = zip.finish().context("finish encoding zip archive")?; - let encoder = tar.into_inner().context("get encoder from tar archive")?; - let bytes = encoder.finish().context("finish up encoder")?; + r.into_inner() + } else { + debug!("making tar archive"); + let encoder = GzEncoder::new(Vec::new(), Compression::new(3)); + let mut tar = Builder::new(encoder); + for (path, name) in archive_files { + debug!("Packing {path:?}"); + tar.append_path_with_name(path, name)?; + } + let encoder = tar.into_inner().context("get encoder from tar archive")?; + + encoder.finish().context("finish encoding tar archive")? + }; debug!("Archive size: {} bytes", bytes.len()); Ok(bytes) @@ -2418,10 +2459,12 @@ pub enum CommandOutcome { mod tests { use flate2::read::GzDecoder; use tar::Archive; + use zip::ZipArchive; use crate::args::{DeployArgs, ProjectArgs, SecretsArgs}; use crate::Shuttle; use std::fs::{self, canonicalize}; + use std::io::Cursor; use std::path::PathBuf; pub fn path_from_workspace_root(path: &str) -> PathBuf { @@ -2432,32 +2475,43 @@ mod tests { dunce::canonicalize(path).unwrap() } - fn get_archive_entries(project_args: ProjectArgs, deploy_args: DeployArgs) -> Vec { + fn get_archive_entries( + project_args: ProjectArgs, + deploy_args: DeployArgs, + zip: bool, + ) -> Vec { let mut shuttle = Shuttle::new().unwrap(); shuttle.load_project(&project_args).unwrap(); let archive = shuttle - .make_archive(deploy_args.secret_args.secrets) + .make_archive(deploy_args.secret_args.secrets, zip) .unwrap(); - let tar = GzDecoder::new(&archive[..]); - let mut archive = Archive::new(tar); + if zip { + let mut zip = ZipArchive::new(Cursor::new(archive)).unwrap(); + (0..zip.len()) + .map(|i| zip.by_index(i).unwrap().name().to_owned()) + .collect() + } else { + let tar = GzDecoder::new(&archive[..]); + let mut archive = Archive::new(tar); - archive - .entries() - .unwrap() - .map(|entry| { - entry - .unwrap() - .path() - .unwrap() - .components() - .skip(1) - .collect::() - .display() - .to_string() - }) - .collect() + archive + .entries() + .unwrap() + .map(|entry| { + entry + .unwrap() + .path() + .unwrap() + .components() + .skip(1) + .collect::() + .display() + .to_string() + }) + .collect() + } } #[test] @@ -2481,29 +2535,32 @@ mod tests { working_directory: working_directory.clone(), name: Some("archiving-test".to_owned()), }; - let mut entries = get_archive_entries(project_args.clone(), Default::default()); + let mut entries = get_archive_entries(project_args.clone(), Default::default(), false); entries.sort(); - assert_eq!( - entries, - vec![ - ".gitignore", - ".ignore", - "Cargo.toml", - "Secrets.toml", // always included by default - "Secrets.toml.example", - "Shuttle.toml", - "asset1", // normal file - "asset2", // .gitignore'd, but included in Shuttle.toml - // asset3 is .ignore'd - "asset4", // .gitignore'd, but un-ignored in .ignore - "asset5", // .ignore'd, but included in Shuttle.toml - "dist/dist1", // .gitignore'd, but included in Shuttle.toml - "nested/static/nested1", // normal file - // nested/static/nestedignore is .gitignore'd - "src/main.rs", - ] - ); + let expected = vec![ + ".gitignore", + ".ignore", + "Cargo.toml", + "Secrets.toml", // always included by default + "Secrets.toml.example", + "Shuttle.toml", + "asset1", // normal file + "asset2", // .gitignore'd, but included in Shuttle.toml + // asset3 is .ignore'd + "asset4", // .gitignore'd, but un-ignored in .ignore + "asset5", // .ignore'd, but included in Shuttle.toml + "dist/dist1", // .gitignore'd, but included in Shuttle.toml + "nested/static/nested1", // normal file + // nested/static/nestedignore is .gitignore'd + "src/main.rs", + ]; + assert_eq!(entries, expected); + + // check that zip behaves the same way + let mut entries = get_archive_entries(project_args.clone(), Default::default(), true); + entries.sort(); + assert_eq!(entries, expected); fs::remove_file(working_directory.join("Secrets.toml")).unwrap(); let mut entries = get_archive_entries( @@ -2514,6 +2571,7 @@ mod tests { }, ..Default::default() }, + false, ); entries.sort(); diff --git a/common/src/models/deployment.rs b/common/src/models/deployment.rs index 877cafe25..475d4e37f 100644 --- a/common/src/models/deployment.rs +++ b/common/src/models/deployment.rs @@ -75,7 +75,7 @@ impl EcsResponse { self.uri ) }) - .unwrap_or(String::new()); + .unwrap_or_default(); // Stringify the state. let latest_state = format!( @@ -269,7 +269,11 @@ pub fn get_deployments_table( #[derive(Default, Deserialize, Serialize)] pub struct DeploymentRequest { + /// Alpha: tar archive. Beta: zip archive. pub data: Vec, + /// The cargo package name to compile and run. Required on beta. + pub package_name: Option, + /// Ignored on beta. pub no_test: bool, pub git_commit_id: Option, pub git_commit_msg: Option, diff --git a/service/src/builder.rs b/service/src/builder.rs index e60eaa3a4..a6f86828a 100644 --- a/service/src/builder.rs +++ b/service/src/builder.rs @@ -3,7 +3,7 @@ use std::path::{Path, PathBuf}; use std::process::Stdio; use anyhow::{anyhow, bail, Context}; -use cargo_metadata::Package; +use cargo_metadata::{Metadata, Package}; use shuttle_common::constants::RUNTIME_NAME; use tokio::io::AsyncBufReadExt; use tracing::{debug, error, info, trace}; @@ -99,12 +99,30 @@ pub async fn build_workspace( } notification.abort(); + let metadata = async_cargo_metadata(manifest_path.as_path()).await?; + let packages = find_shuttle_packages(&metadata)?; + + let services = compile( + packages, + release_mode, + project_path.clone(), + metadata.target_directory.clone(), + deployment, + tx.clone(), + ) + .await?; + trace!("alpha packages compiled"); + + Ok(services) +} + +pub async fn async_cargo_metadata(manifest_path: &Path) -> anyhow::Result { let metadata = { // Modified implementaion of `cargo_metadata::MetadataCommand::exec` (from v0.15.3). // Uses tokio Command instead of std, to make this operation non-blocking. let mut cmd = tokio::process::Command::from( cargo_metadata::MetadataCommand::new() - .manifest_path(&manifest_path) + .manifest_path(manifest_path) .cargo_command(), ); @@ -120,11 +138,13 @@ pub async fn build_workspace( .ok_or(cargo_metadata::Error::NoJson)?; cargo_metadata::MetadataCommand::parse(json)? }; - trace!("Cargo metadata parsed"); - let mut alpha_packages = Vec::new(); + Ok(metadata) +} +pub fn find_shuttle_packages(metadata: &Metadata) -> anyhow::Result> { + let mut packages = Vec::new(); for member in metadata.workspace_packages() { // skip non-Shuttle-related crates if !member @@ -141,23 +161,12 @@ pub async fn build_workspace( .map(|d| format!("{} '{}'", d.name, d.req)) .collect::>(); shuttle_deps.sort(); - info!(name = member.name, deps = ?shuttle_deps, "Compiling workspace member with shuttle dependencies"); + info!(name = member.name, deps = ?shuttle_deps, "Found workspace member with shuttle dependencies"); ensure_binary(member)?; - alpha_packages.push(member); + packages.push(member.to_owned()); } - let services = compile( - alpha_packages, - release_mode, - project_path.clone(), - metadata.target_directory.clone(), - deployment, - tx.clone(), - ) - .await?; - trace!("alpha packages compiled"); - - Ok(services) + Ok(packages) } // Only used in deployer @@ -191,7 +200,7 @@ fn ensure_binary(package: &Package) -> anyhow::Result<()> { } async fn compile( - packages: Vec<&Package>, + packages: Vec, release_mode: bool, project_path: PathBuf, target_path: impl Into,