From b17b3a19193779a63a6cbe914aaeb71877ede03b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oddbj=C3=B8rn=20Gr=C3=B8dem?= <29732646+oddgrd@users.noreply.github.com> Date: Mon, 16 Jan 2023 10:34:19 +0100 Subject: [PATCH] feat: get runtime binary from cargo install (#578) * feat: get runtime binary from cargo install * refactor: remove build.rs * refactor: determine environment with debug assertions * ci: comment out cargo-sort installation * fix: clippy * feat: use cargo home dir, install runtime in prepare.sh * fix: unused import * refactor: build from local version in prepare.sh * fix: local debug run installing from incorrect path * feat: canonicalize path to debug runtime * feat: set release runtime install branch to prod * refactor: move secondary protoc install to common stage --- .circleci/config.yml | 4 +-- Cargo.lock | 1 + Containerfile | 5 ++++ cargo-shuttle/build.rs | 17 ------------ cargo-shuttle/src/lib.rs | 3 --- deployer/build.rs | 17 ------------ deployer/prepare.sh | 3 +++ deployer/src/main.rs | 3 --- proto/Cargo.toml | 1 + proto/src/lib.rs | 58 +++++++++++++++++++++++----------------- 10 files changed, 46 insertions(+), 66 deletions(-) delete mode 100644 cargo-shuttle/build.rs delete mode 100644 deployer/build.rs diff --git a/.circleci/config.yml b/.circleci/config.yml index d14ee3eca..910fb6085 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -148,9 +148,9 @@ jobs: - restore-cargo-cache - install-protoc - run: cargo fmt --all --check - - run: cargo install cargo-sort # TODO: this is incompatible with workspace inheritance, uncomment when # https://github.com/DevinR528/cargo-sort/pull/29 is merged + # - run: cargo install cargo-sort # - run: cargo sort --check --workspace - run: cargo check --workspace --all-targets - save-cargo-cache @@ -185,9 +185,9 @@ jobs: - install-protoc - apply-patches - run: cargo fmt --all --check --manifest-path << parameters.path >>/Cargo.toml - - run: cargo install cargo-sort # TODO: this is incompatible with workspace inheritance, uncomment when # https://github.com/DevinR528/cargo-sort/pull/29 is merged + # - run: cargo install cargo-sort # - run: cargo sort --check << parameters.path >> - run: | cargo clippy --tests \ diff --git a/Cargo.lock b/Cargo.lock index 42b9b445d..e94855c62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6364,6 +6364,7 @@ version = "0.8.0" dependencies = [ "anyhow", "chrono", + "home", "prost", "prost-types", "shuttle-common", diff --git a/Containerfile b/Containerfile index 9c382250a..015bfae3f 100644 --- a/Containerfile +++ b/Containerfile @@ -31,6 +31,11 @@ ARG RUSTUP_TOOLCHAIN FROM rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-common RUN apt-get update &&\ apt-get install -y curl +# download protoc binary and unzip it in usr/bin +RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.9/protoc-21.9-linux-x86_64.zip &&\ + unzip -o protoc-21.9-linux-x86_64.zip -d /usr bin/protoc &&\ + unzip -o protoc-21.9-linux-x86_64.zip -d /usr/ 'include/*' &&\ + rm -f protoc-21.9-linux-x86_64.zip RUN rustup component add rust-src COPY --from=cache /build/ /usr/src/shuttle/ diff --git a/cargo-shuttle/build.rs b/cargo-shuttle/build.rs deleted file mode 100644 index 4503a843d..000000000 --- a/cargo-shuttle/build.rs +++ /dev/null @@ -1,17 +0,0 @@ -use std::{env, process::Command}; - -fn main() { - println!("cargo:rerun-if-changed=../runtime"); - - // Build binary for runtime so that it can be embedded in the binary for the cli - let out_dir = env::var_os("OUT_DIR").unwrap(); - Command::new("cargo") - .arg("build") - .arg("--package") - .arg("shuttle-runtime") - .arg("--target-dir") - .arg(out_dir) - .arg("--release") - .output() - .expect("failed to build the shuttle runtime"); -} diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs index 56e4edea9..e6851fb4c 100644 --- a/cargo-shuttle/src/lib.rs +++ b/cargo-shuttle/src/lib.rs @@ -39,8 +39,6 @@ use uuid::Uuid; use crate::args::{DeploymentCommand, ProjectCommand}; use crate::client::Client; -const BINARY_BYTES: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/release/shuttle-runtime")); - pub struct Shuttle { ctx: RequestContext, } @@ -412,7 +410,6 @@ impl Shuttle { run_args.port + 1, )); let (mut runtime, mut runtime_client) = runtime::start( - BINARY_BYTES, is_wasm, runtime::StorageManagerType::WorkingDir(working_directory.to_path_buf()), &format!("http://localhost:{}", run_args.port + 1), diff --git a/deployer/build.rs b/deployer/build.rs deleted file mode 100644 index ad2a67032..000000000 --- a/deployer/build.rs +++ /dev/null @@ -1,17 +0,0 @@ -use std::{env, process::Command}; - -fn main() { - println!("cargo:rerun-if-changed=../runtime"); - - // Build binary for runtime so that it can be embedded in the binary for deployer - let out_dir = env::var_os("OUT_DIR").unwrap(); - Command::new("cargo") - .arg("build") - .arg("--package") - .arg("shuttle-runtime") - .arg("--target-dir") - .arg(out_dir) - .arg("--release") - .output() - .expect("failed to build the shuttle runtime"); -} diff --git a/deployer/prepare.sh b/deployer/prepare.sh index 70eac8a23..119b2e834 100755 --- a/deployer/prepare.sh +++ b/deployer/prepare.sh @@ -15,6 +15,9 @@ shuttle-shared-db = { path = "/usr/src/shuttle/resources/shared-db" } shuttle-secrets = { path = "/usr/src/shuttle/resources/secrets" } shuttle-static-folder = { path = "/usr/src/shuttle/resources/static-folder" }' > $CARGO_HOME/config.toml +# Install the shuttle runtime +cargo install shuttle-runtime --path "/usr/src/shuttle/runtime" + # Make future crates requests to our own mirror echo ' [source.shuttle-crates-io-mirror] diff --git a/deployer/src/main.rs b/deployer/src/main.rs index 846f5c30c..d14296fef 100644 --- a/deployer/src/main.rs +++ b/deployer/src/main.rs @@ -9,8 +9,6 @@ use tracing::{error, trace}; use tracing_subscriber::prelude::*; use tracing_subscriber::{fmt, EnvFilter}; -const BINARY_BYTES: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/release/shuttle-runtime")); - // The `multi_thread` is needed to prevent a deadlock in shuttle_service::loader::build_crate() which spawns two threads // Without this, both threads just don't start up #[tokio::main(flavor = "multi_thread")] @@ -41,7 +39,6 @@ async fn main() { .init(); let (mut runtime, mut runtime_client) = runtime::start( - BINARY_BYTES, false, runtime::StorageManagerType::Artifacts(args.artifacts_path.clone()), &args.provisioner_address.uri().to_string(), diff --git a/proto/Cargo.toml b/proto/Cargo.toml index bc1680850..9e44fa415 100644 --- a/proto/Cargo.toml +++ b/proto/Cargo.toml @@ -9,6 +9,7 @@ publish = false [dependencies] anyhow = { workspace = true } chrono = { workspace = true } +home = "0.5.4" prost = "0.11.2" prost-types = "0.11.0" tokio = { version = "1.22.0", features = ["process"] } diff --git a/proto/src/lib.rs b/proto/src/lib.rs index ae78a8487..887e54a81 100644 --- a/proto/src/lib.rs +++ b/proto/src/lib.rs @@ -96,10 +96,8 @@ pub mod provisioner { pub mod runtime { use std::{ - env::temp_dir, - fs::OpenOptions, - io::Write, path::PathBuf, + process::Command, time::{Duration, SystemTime}, }; @@ -240,7 +238,6 @@ pub mod runtime { } pub async fn start( - binary_bytes: &[u8], wasm: bool, storage_manager_type: StorageManagerType, provisioner_address: &str, @@ -252,7 +249,7 @@ pub mod runtime { StorageManagerType::WorkingDir(path) => ("working-dir", path), }; - let runtime_executable = get_runtime_executable(binary_bytes); + let runtime_executable = get_runtime_executable(); let runtime = process::Command::new(runtime_executable) .args([ @@ -283,27 +280,40 @@ pub mod runtime { Ok((runtime, runtime_client)) } - fn get_runtime_executable(binary_bytes: &[u8]) -> PathBuf { - let tmp_dir = temp_dir(); - - let path = tmp_dir.join("shuttle-runtime"); - let mut open_options = OpenOptions::new(); - open_options.write(true).create(true).truncate(true); - - #[cfg(target_family = "unix")] - { - use std::os::unix::prelude::OpenOptionsExt; - - open_options.mode(0o755); + fn get_runtime_executable() -> PathBuf { + // When this library is compiled in debug mode with `cargo run --bin cargo-shuttle`, + // install the checked-out local version of `shuttle-runtime + if cfg!(debug_assertions) { + // Path to cargo-shuttle + let manifest_dir = env!("CARGO_MANIFEST_DIR"); + + // Canonicalized path to shuttle-runtime + let path = std::fs::canonicalize(format!("{manifest_dir}/../runtime")) + .expect("failed to canonicalize path to runtime"); + + Command::new("cargo") + .arg("install") + .arg("shuttle-runtime") + .arg("--path") + .arg(path) + .output() + .expect("failed to install the shuttle runtime"); + // When this library is compiled in release mode with `cargo install cargo-shuttle`, + // install the latest released `shuttle-runtime` + } else { + Command::new("cargo") + .arg("install") + .arg("shuttle-runtime") + .arg("--git") + .arg("https://github.com/shuttle-hq/shuttle") + .arg("--branch") + .arg("production") + .output() + .expect("failed to install the shuttle runtime"); } - let mut file = open_options - .open(&path) - .expect("to create runtime executable file"); - - file.write_all(binary_bytes) - .expect("to write out binary file"); + let cargo_home = home::cargo_home().expect("failed to find cargo home directory"); - path + cargo_home.join("bin/shuttle-runtime") } }