From 132aa76c4be4bd226718f73cb22cb7b538fd8fe8 Mon Sep 17 00:00:00 2001
From: oddgrd <29732646+oddgrd@users.noreply.github.com>
Date: Thu, 12 Jan 2023 14:42:05 +0100
Subject: [PATCH 01/12] feat: get runtime binary from cargo install

---
 cargo-shuttle/src/lib.rs |  3 ---
 deployer/src/main.rs     |  3 ---
 proto/src/lib.rs         | 49 +++++++++++++++++++++++-----------------
 3 files changed, 28 insertions(+), 27 deletions(-)

diff --git a/cargo-shuttle/src/lib.rs b/cargo-shuttle/src/lib.rs
index 56e4edea9..e6851fb4c 100644
--- a/cargo-shuttle/src/lib.rs
+++ b/cargo-shuttle/src/lib.rs
@@ -39,8 +39,6 @@ use uuid::Uuid;
 use crate::args::{DeploymentCommand, ProjectCommand};
 use crate::client::Client;
 
-const BINARY_BYTES: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/release/shuttle-runtime"));
-
 pub struct Shuttle {
     ctx: RequestContext,
 }
@@ -412,7 +410,6 @@ impl Shuttle {
             run_args.port + 1,
         ));
         let (mut runtime, mut runtime_client) = runtime::start(
-            BINARY_BYTES,
             is_wasm,
             runtime::StorageManagerType::WorkingDir(working_directory.to_path_buf()),
             &format!("http://localhost:{}", run_args.port + 1),
diff --git a/deployer/src/main.rs b/deployer/src/main.rs
index 846f5c30c..d14296fef 100644
--- a/deployer/src/main.rs
+++ b/deployer/src/main.rs
@@ -9,8 +9,6 @@ use tracing::{error, trace};
 use tracing_subscriber::prelude::*;
 use tracing_subscriber::{fmt, EnvFilter};
 
-const BINARY_BYTES: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/release/shuttle-runtime"));
-
 // The `multi_thread` is needed to prevent a deadlock in shuttle_service::loader::build_crate() which spawns two threads
 // Without this, both threads just don't start up
 #[tokio::main(flavor = "multi_thread")]
@@ -41,7 +39,6 @@ async fn main() {
         .init();
 
     let (mut runtime, mut runtime_client) = runtime::start(
-        BINARY_BYTES,
         false,
         runtime::StorageManagerType::Artifacts(args.artifacts_path.clone()),
         &args.provisioner_address.uri().to_string(),
diff --git a/proto/src/lib.rs b/proto/src/lib.rs
index ae78a8487..edf4818a4 100644
--- a/proto/src/lib.rs
+++ b/proto/src/lib.rs
@@ -97,9 +97,8 @@ pub mod provisioner {
 pub mod runtime {
     use std::{
         env::temp_dir,
-        fs::OpenOptions,
-        io::Write,
         path::PathBuf,
+        process::Command,
         time::{Duration, SystemTime},
     };
 
@@ -240,7 +239,6 @@ pub mod runtime {
     }
 
     pub async fn start(
-        binary_bytes: &[u8],
         wasm: bool,
         storage_manager_type: StorageManagerType,
         provisioner_address: &str,
@@ -252,7 +250,7 @@ pub mod runtime {
             StorageManagerType::WorkingDir(path) => ("working-dir", path),
         };
 
-        let runtime_executable = get_runtime_executable(binary_bytes);
+        let runtime_executable = get_runtime_executable();
 
         let runtime = process::Command::new(runtime_executable)
             .args([
@@ -283,26 +281,35 @@ pub mod runtime {
         Ok((runtime, runtime_client))
     }
 
-    fn get_runtime_executable(binary_bytes: &[u8]) -> PathBuf {
+    fn get_runtime_executable() -> PathBuf {
         let tmp_dir = temp_dir();
 
-        let path = tmp_dir.join("shuttle-runtime");
-        let mut open_options = OpenOptions::new();
-        open_options.write(true).create(true).truncate(true);
-
-        #[cfg(target_family = "unix")]
-        {
-            use std::os::unix::prelude::OpenOptionsExt;
-
-            open_options.mode(0o755);
+        let dev = std::env::var("SHUTTLE_DEV");
+
+        if dev.is_ok() {
+            Command::new("cargo")
+                .arg("install")
+                .arg("shuttle-runtime")
+                .arg("--root")
+                .arg(&tmp_dir)
+                .arg("--path")
+                .arg("../../runtime")
+                .output()
+                .expect("failed to install the shuttle runtime");
+        } else {
+            Command::new("cargo")
+                .arg("install")
+                .arg("shuttle-runtime")
+                .arg("--root")
+                .arg(&tmp_dir)
+                .arg("--git")
+                .arg("https://github.com/shuttle-hq/shuttle")
+                .arg("--branch")
+                .arg("shuttle-next")
+                .output()
+                .expect("failed to install the shuttle runtime");
         }
-
-        let mut file = open_options
-            .open(&path)
-            .expect("to create runtime executable file");
-
-        file.write_all(binary_bytes)
-            .expect("to write out binary file");
+        let path = tmp_dir.join("bin/shuttle-runtime");
 
         path
     }

From 4ea6c87e0693c91d0b1e3abeb51dc65b3e8c6aa6 Mon Sep 17 00:00:00 2001
From: oddgrd <29732646+oddgrd@users.noreply.github.com>
Date: Thu, 12 Jan 2023 14:54:48 +0100
Subject: [PATCH 02/12] refactor: remove build.rs

---
 cargo-shuttle/build.rs | 17 -----------------
 deployer/build.rs      | 17 -----------------
 2 files changed, 34 deletions(-)
 delete mode 100644 cargo-shuttle/build.rs
 delete mode 100644 deployer/build.rs

diff --git a/cargo-shuttle/build.rs b/cargo-shuttle/build.rs
deleted file mode 100644
index 4503a843d..000000000
--- a/cargo-shuttle/build.rs
+++ /dev/null
@@ -1,17 +0,0 @@
-use std::{env, process::Command};
-
-fn main() {
-    println!("cargo:rerun-if-changed=../runtime");
-
-    // Build binary for runtime so that it can be embedded in the binary for the cli
-    let out_dir = env::var_os("OUT_DIR").unwrap();
-    Command::new("cargo")
-        .arg("build")
-        .arg("--package")
-        .arg("shuttle-runtime")
-        .arg("--target-dir")
-        .arg(out_dir)
-        .arg("--release")
-        .output()
-        .expect("failed to build the shuttle runtime");
-}
diff --git a/deployer/build.rs b/deployer/build.rs
deleted file mode 100644
index ad2a67032..000000000
--- a/deployer/build.rs
+++ /dev/null
@@ -1,17 +0,0 @@
-use std::{env, process::Command};
-
-fn main() {
-    println!("cargo:rerun-if-changed=../runtime");
-
-    // Build binary for runtime so that it can be embedded in the binary for deployer
-    let out_dir = env::var_os("OUT_DIR").unwrap();
-    Command::new("cargo")
-        .arg("build")
-        .arg("--package")
-        .arg("shuttle-runtime")
-        .arg("--target-dir")
-        .arg(out_dir)
-        .arg("--release")
-        .output()
-        .expect("failed to build the shuttle runtime");
-}

From 5377b0699d9dedf6702cda82ff75ea01476ba1c5 Mon Sep 17 00:00:00 2001
From: oddgrd <29732646+oddgrd@users.noreply.github.com>
Date: Thu, 12 Jan 2023 15:07:15 +0100
Subject: [PATCH 03/12] refactor: determine environment with debug assertions

---
 proto/src/lib.rs | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/proto/src/lib.rs b/proto/src/lib.rs
index edf4818a4..f202ecd34 100644
--- a/proto/src/lib.rs
+++ b/proto/src/lib.rs
@@ -284,9 +284,7 @@ pub mod runtime {
     fn get_runtime_executable() -> PathBuf {
         let tmp_dir = temp_dir();
 
-        let dev = std::env::var("SHUTTLE_DEV");
-
-        if dev.is_ok() {
+        if cfg!(debug_assertions) {
             Command::new("cargo")
                 .arg("install")
                 .arg("shuttle-runtime")

From f63893ea024c72a685a9cc31716eaacc9dd0f2df Mon Sep 17 00:00:00 2001
From: oddgrd <29732646+oddgrd@users.noreply.github.com>
Date: Thu, 12 Jan 2023 15:13:19 +0100
Subject: [PATCH 04/12] ci: comment out cargo-sort installation

---
 .circleci/config.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 3e9650676..e8474dae7 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -148,9 +148,9 @@ jobs:
       - restore-cargo-cache
       - install-protoc
       - run: cargo fmt --all --check
-      - run: cargo install cargo-sort
       # TODO: this is incompatible with workspace inheritance, uncomment when 
       # https://github.com/DevinR528/cargo-sort/pull/29 is merged
+      # - run: cargo install cargo-sort
       # - run: cargo sort --check --workspace
       - run: cargo check --workspace --all-targets
       - save-cargo-cache
@@ -185,9 +185,9 @@ jobs:
       - install-protoc
       - apply-patches
       - run: cargo fmt --all --check --manifest-path << parameters.path >>/Cargo.toml
-      - run: cargo install cargo-sort
       # TODO: this is incompatible with workspace inheritance, uncomment when 
       # https://github.com/DevinR528/cargo-sort/pull/29 is merged
+      # - run: cargo install cargo-sort
       # - run: cargo sort --check << parameters.path >>
       - run: |
           cargo clippy --tests \

From ad5412b95366fffb9018fe724421b1e44817aba9 Mon Sep 17 00:00:00 2001
From: oddgrd <29732646+oddgrd@users.noreply.github.com>
Date: Thu, 12 Jan 2023 15:19:28 +0100
Subject: [PATCH 05/12] fix: clippy

---
 proto/src/lib.rs | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/proto/src/lib.rs b/proto/src/lib.rs
index f202ecd34..300a728b7 100644
--- a/proto/src/lib.rs
+++ b/proto/src/lib.rs
@@ -307,8 +307,7 @@ pub mod runtime {
                 .output()
                 .expect("failed to install the shuttle runtime");
         }
-        let path = tmp_dir.join("bin/shuttle-runtime");
 
-        path
+        tmp_dir.join("bin/shuttle-runtime")
     }
 }

From 6970e617023a5d6f7fb7d8ce70c054bf44376dab Mon Sep 17 00:00:00 2001
From: oddgrd <29732646+oddgrd@users.noreply.github.com>
Date: Thu, 12 Jan 2023 15:52:19 +0100
Subject: [PATCH 06/12] feat: use cargo home dir, install runtime in prepare.sh

---
 Cargo.lock          |  1 +
 deployer/prepare.sh |  3 +++
 proto/Cargo.toml    |  1 +
 proto/src/lib.rs    | 10 +++-------
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index ff2cff0bd..47c148be5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6350,6 +6350,7 @@ version = "0.8.0"
 dependencies = [
  "anyhow",
  "chrono",
+ "home",
  "prost",
  "prost-types",
  "shuttle-common",
diff --git a/deployer/prepare.sh b/deployer/prepare.sh
index 70eac8a23..8e784c06b 100755
--- a/deployer/prepare.sh
+++ b/deployer/prepare.sh
@@ -15,6 +15,9 @@ shuttle-shared-db = { path = "/usr/src/shuttle/resources/shared-db" }
 shuttle-secrets = { path = "/usr/src/shuttle/resources/secrets" }
 shuttle-static-folder = { path = "/usr/src/shuttle/resources/static-folder" }' > $CARGO_HOME/config.toml
 
+# Install the shuttle runtime
+cargo install shuttle-runtime --git "https://github.com/shuttle-hq/shuttle" --branch "shuttle-next"
+
 # Make future crates requests to our own mirror
 echo '
 [source.shuttle-crates-io-mirror]
diff --git a/proto/Cargo.toml b/proto/Cargo.toml
index ab998ffa0..3f98eeeb2 100644
--- a/proto/Cargo.toml
+++ b/proto/Cargo.toml
@@ -9,6 +9,7 @@ publish = false
 [dependencies]
 anyhow = { workspace = true }
 chrono = { workspace = true }
+home = "0.5.4"
 prost = "0.11.2"
 prost-types = "0.11.0"
 tokio = { version = "1.22.0", features = ["process"] }
diff --git a/proto/src/lib.rs b/proto/src/lib.rs
index 300a728b7..65e077af5 100644
--- a/proto/src/lib.rs
+++ b/proto/src/lib.rs
@@ -282,14 +282,10 @@ pub mod runtime {
     }
 
     fn get_runtime_executable() -> PathBuf {
-        let tmp_dir = temp_dir();
-
         if cfg!(debug_assertions) {
             Command::new("cargo")
                 .arg("install")
                 .arg("shuttle-runtime")
-                .arg("--root")
-                .arg(&tmp_dir)
                 .arg("--path")
                 .arg("../../runtime")
                 .output()
@@ -298,8 +294,6 @@ pub mod runtime {
             Command::new("cargo")
                 .arg("install")
                 .arg("shuttle-runtime")
-                .arg("--root")
-                .arg(&tmp_dir)
                 .arg("--git")
                 .arg("https://github.com/shuttle-hq/shuttle")
                 .arg("--branch")
@@ -308,6 +302,8 @@ pub mod runtime {
                 .expect("failed to install the shuttle runtime");
         }
 
-        tmp_dir.join("bin/shuttle-runtime")
+        let cargo_home = home::cargo_home().expect("failed to find cargo home directory");
+
+        cargo_home.join("bin/shuttle-runtime")
     }
 }

From f4c2c1d3cd3b2412ab05f90a81a4b3b83ac9d88e Mon Sep 17 00:00:00 2001
From: oddgrd <29732646+oddgrd@users.noreply.github.com>
Date: Thu, 12 Jan 2023 15:53:57 +0100
Subject: [PATCH 07/12] fix: unused import

---
 proto/src/lib.rs | 1 -
 1 file changed, 1 deletion(-)

diff --git a/proto/src/lib.rs b/proto/src/lib.rs
index 65e077af5..32bb104db 100644
--- a/proto/src/lib.rs
+++ b/proto/src/lib.rs
@@ -96,7 +96,6 @@ pub mod provisioner {
 
 pub mod runtime {
     use std::{
-        env::temp_dir,
         path::PathBuf,
         process::Command,
         time::{Duration, SystemTime},

From 967d1d389858792f17cf5856ec82951882f659de Mon Sep 17 00:00:00 2001
From: oddgrd <29732646+oddgrd@users.noreply.github.com>
Date: Fri, 13 Jan 2023 09:11:39 +0100
Subject: [PATCH 08/12] refactor: build from local version in prepare.sh

---
 Containerfile       | 4 ++++
 deployer/prepare.sh | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/Containerfile b/Containerfile
index 9c382250a..cfbef90ac 100644
--- a/Containerfile
+++ b/Containerfile
@@ -37,6 +37,10 @@ COPY --from=cache /build/ /usr/src/shuttle/
 FROM shuttle-common
 ARG folder
 COPY ${folder}/prepare.sh /prepare.sh
+RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.9/protoc-21.9-linux-x86_64.zip &&\
+    unzip -o protoc-21.9-linux-x86_64.zip -d /usr bin/protoc &&\
+    unzip -o protoc-21.9-linux-x86_64.zip -d /usr/ 'include/*' &&\
+    rm -f protoc-21.9-linux-x86_64.zip
 RUN /prepare.sh
 COPY --from=builder /build/target/debug/shuttle-${folder} /usr/local/bin/service
 ARG RUSTUP_TOOLCHAIN
diff --git a/deployer/prepare.sh b/deployer/prepare.sh
index 8e784c06b..119b2e834 100755
--- a/deployer/prepare.sh
+++ b/deployer/prepare.sh
@@ -16,7 +16,7 @@ shuttle-secrets = { path = "/usr/src/shuttle/resources/secrets" }
 shuttle-static-folder = { path = "/usr/src/shuttle/resources/static-folder" }' > $CARGO_HOME/config.toml
 
 # Install the shuttle runtime
-cargo install shuttle-runtime --git "https://github.com/shuttle-hq/shuttle" --branch "shuttle-next"
+cargo install shuttle-runtime --path "/usr/src/shuttle/runtime" 
 
 # Make future crates requests to our own mirror
 echo '

From 49f1ce8e3a6bda7ecb0fbcfceda2989c14be23f9 Mon Sep 17 00:00:00 2001
From: oddgrd <29732646+oddgrd@users.noreply.github.com>
Date: Fri, 13 Jan 2023 09:46:07 +0100
Subject: [PATCH 09/12] fix: local debug run installing from incorrect path

---
 proto/src/lib.rs | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/proto/src/lib.rs b/proto/src/lib.rs
index 32bb104db..7f152b149 100644
--- a/proto/src/lib.rs
+++ b/proto/src/lib.rs
@@ -282,11 +282,13 @@ pub mod runtime {
 
     fn get_runtime_executable() -> PathBuf {
         if cfg!(debug_assertions) {
+            let manifest_dir = env!("CARGO_MANIFEST_DIR");
+
             Command::new("cargo")
                 .arg("install")
                 .arg("shuttle-runtime")
                 .arg("--path")
-                .arg("../../runtime")
+                .arg(format!("{manifest_dir}/../runtime"))
                 .output()
                 .expect("failed to install the shuttle runtime");
         } else {

From ad4814c93d06b5cddf5464b0c587ae5ffcfc5f43 Mon Sep 17 00:00:00 2001
From: oddgrd <29732646+oddgrd@users.noreply.github.com>
Date: Fri, 13 Jan 2023 11:40:50 +0100
Subject: [PATCH 10/12] feat: canonicalize path to debug runtime

---
 proto/src/lib.rs | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/proto/src/lib.rs b/proto/src/lib.rs
index 7f152b149..9a9d883d1 100644
--- a/proto/src/lib.rs
+++ b/proto/src/lib.rs
@@ -281,16 +281,25 @@ pub mod runtime {
     }
 
     fn get_runtime_executable() -> PathBuf {
+        // When this library is compiled in debug mode with `cargo run --bin cargo-shuttle`,
+        // install the checked-out local version of `shuttle-runtime
         if cfg!(debug_assertions) {
+            // Path to cargo-shuttle
             let manifest_dir = env!("CARGO_MANIFEST_DIR");
 
+            // Canonicalized path to shuttle-runtime
+            let path = std::fs::canonicalize(format!("{manifest_dir}/../runtime"))
+                .expect("failed to canonicalize path to runtime");
+
             Command::new("cargo")
                 .arg("install")
                 .arg("shuttle-runtime")
                 .arg("--path")
-                .arg(format!("{manifest_dir}/../runtime"))
+                .arg(path)
                 .output()
                 .expect("failed to install the shuttle runtime");
+        // When this library is compiled in release mode with `cargo install cargo-shuttle`,
+        // install the latest released `shuttle-runtime`
         } else {
             Command::new("cargo")
                 .arg("install")

From 530677c40a73141049f01dd45dd51b1f1528486d Mon Sep 17 00:00:00 2001
From: oddgrd <29732646+oddgrd@users.noreply.github.com>
Date: Fri, 13 Jan 2023 11:46:00 +0100
Subject: [PATCH 11/12] feat: set release runtime install branch to prod

---
 proto/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/proto/src/lib.rs b/proto/src/lib.rs
index 9a9d883d1..887e54a81 100644
--- a/proto/src/lib.rs
+++ b/proto/src/lib.rs
@@ -307,7 +307,7 @@ pub mod runtime {
                 .arg("--git")
                 .arg("https://github.com/shuttle-hq/shuttle")
                 .arg("--branch")
-                .arg("shuttle-next")
+                .arg("production")
                 .output()
                 .expect("failed to install the shuttle runtime");
         }

From 32026dc6b69b697c5f56ff969924131ea698881f Mon Sep 17 00:00:00 2001
From: oddgrd <29732646+oddgrd@users.noreply.github.com>
Date: Mon, 16 Jan 2023 10:24:30 +0100
Subject: [PATCH 12/12] refactor: move secondary protoc install to common stage

---
 Containerfile | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/Containerfile b/Containerfile
index cfbef90ac..015bfae3f 100644
--- a/Containerfile
+++ b/Containerfile
@@ -31,16 +31,17 @@ ARG RUSTUP_TOOLCHAIN
 FROM rust:${RUSTUP_TOOLCHAIN}-buster as shuttle-common
 RUN apt-get update &&\
     apt-get install -y curl
+# download protoc binary and unzip it in usr/bin
+RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.9/protoc-21.9-linux-x86_64.zip &&\
+    unzip -o protoc-21.9-linux-x86_64.zip -d /usr bin/protoc &&\
+    unzip -o protoc-21.9-linux-x86_64.zip -d /usr/ 'include/*' &&\
+    rm -f protoc-21.9-linux-x86_64.zip
 RUN rustup component add rust-src
 COPY --from=cache /build/ /usr/src/shuttle/
 
 FROM shuttle-common
 ARG folder
 COPY ${folder}/prepare.sh /prepare.sh
-RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.9/protoc-21.9-linux-x86_64.zip &&\
-    unzip -o protoc-21.9-linux-x86_64.zip -d /usr bin/protoc &&\
-    unzip -o protoc-21.9-linux-x86_64.zip -d /usr/ 'include/*' &&\
-    rm -f protoc-21.9-linux-x86_64.zip
 RUN /prepare.sh
 COPY --from=builder /build/target/debug/shuttle-${folder} /usr/local/bin/service
 ARG RUSTUP_TOOLCHAIN