From eb1a6cf65ead433e5976176343f3adc1d6dc4174 Mon Sep 17 00:00:00 2001 From: NathanFlurry Date: Mon, 15 Apr 2024 21:18:18 +0000 Subject: [PATCH] chore(dev): respect CARGO_TARGET_DIR in bolt & use non-mounted target in dev container (#675) ## Changes --- .devcontainer/devcontainer.json | 8 ++++++- .../devcontainers/RUST_WORKAROUND.md | 23 +++++++++++++++++++ infra/tf/k8s_cluster_k3d/main.tf | 4 ++-- infra/tf/k8s_cluster_k3d/vars.tf | 4 ++++ lib/bolt/core/src/context/project.rs | 8 +++++++ lib/bolt/core/src/context/service.rs | 3 +-- lib/bolt/core/src/dep/cargo/cli.rs | 2 +- lib/bolt/core/src/dep/k8s/gen.rs | 10 ++++---- lib/bolt/core/src/dep/terraform/gen.rs | 6 +++++ lib/bolt/core/src/tasks/check.rs | 2 +- lib/bolt/core/src/tasks/test.rs | 8 +++++-- lib/bolt/core/src/tasks/up.rs | 2 +- 12 files changed, 65 insertions(+), 15 deletions(-) create mode 100644 docs/infrastructure/devcontainers/RUST_WORKAROUND.md diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 73cac8537..1ce197358 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -26,5 +26,11 @@ "vscode": { "extensions": ["rust-lang.rust-analyzer"] } - } + }, + // docs/infrastructure/devcontainers/RUST_WORKAROUND.md + "containerEnv": { + "CARGO_TARGET_DIR": "/target" + }, + // docs/infrastructure/devcontainers/RUST_WORKAROUND.md + "postCreateCommand": "echo 'export PATH=\"/target/debug:/target/release:${PATH}\"' >> ~/.bashrc" } diff --git a/docs/infrastructure/devcontainers/RUST_WORKAROUND.md b/docs/infrastructure/devcontainers/RUST_WORKAROUND.md new file mode 100644 index 000000000..5983ff6f0 --- /dev/null +++ b/docs/infrastructure/devcontainers/RUST_WORKAROUND.md @@ -0,0 +1,23 @@ +# Rust Workaround + +## Overview + +This issue is specific to Apple Silicon machines. + +Dev containers have a bug with incremental builds which lead to nonsensical Rust errors. + +These are documented in better detail here: + +- https://stackoverflow.com/questions/72448053/rust-incremental-build-not-working-in-vscode-devcontainer +- https://github.com/docker/for-mac/issues/7059 + +## Moving target dir out of mounted dir + +In order to fix this, we move the target path to `/target` which is not mounted to the host system. This way the virtual fs does not trigger issues with Rust. + +This complicates things, since we now compile binaries in non-standard paths. + +This is done by updating `devcontainer.json` to: + +- Overriding `CARGO_TARGET_DIR` to `/target` +- Updating `PATH` to include `/target/debug` and `/target/release` so we can still access `bolt` diff --git a/infra/tf/k8s_cluster_k3d/main.tf b/infra/tf/k8s_cluster_k3d/main.tf index 43eccc017..2ccd3241b 100644 --- a/infra/tf/k8s_cluster_k3d/main.tf +++ b/infra/tf/k8s_cluster_k3d/main.tf @@ -12,8 +12,8 @@ resource "k3d_cluster" "main" { # Mount repository in to k3d so we can access the built binaries volume { - source = var.project_root - destination = "/rivet-src" + source = var.cargo_target_dir + destination = "/target" node_filters = ["server:0"] } diff --git a/infra/tf/k8s_cluster_k3d/vars.tf b/infra/tf/k8s_cluster_k3d/vars.tf index 7c4a0e874..559a16104 100644 --- a/infra/tf/k8s_cluster_k3d/vars.tf +++ b/infra/tf/k8s_cluster_k3d/vars.tf @@ -6,6 +6,10 @@ variable "project_root" { type = string } +variable "cargo_target_dir" { + type = string +} + variable "public_ip" { type = string } diff --git a/lib/bolt/core/src/context/project.rs b/lib/bolt/core/src/context/project.rs index 0422cd4a4..7fd344662 100644 --- a/lib/bolt/core/src/context/project.rs +++ b/lib/bolt/core/src/context/project.rs @@ -2,6 +2,7 @@ use std::{ collections::{HashMap, HashSet}, path::{Path, PathBuf}, process::Command, + str::FromStr, sync::{Arc, Weak}, }; @@ -68,6 +69,13 @@ impl ProjectContextData { pub fn path(&self) -> &Path { self.path.as_path() } + + pub fn cargo_target_dir(&self) -> PathBuf { + std::env::var("CARGO_TARGET_DIR").map_or_else( + |_| self.path().join("target"), + |x| PathBuf::from_str(&x).unwrap(), + ) + } } impl ProjectContextData { diff --git a/lib/bolt/core/src/context/service.rs b/lib/bolt/core/src/context/service.rs index 79dc7ed23..9f012b3fd 100644 --- a/lib/bolt/core/src/context/service.rs +++ b/lib/bolt/core/src/context/service.rs @@ -242,8 +242,7 @@ impl ServiceContextData { pub async fn rust_bin_path(&self, optimization: &BuildOptimization) -> PathBuf { self.project() .await - .path() - .join("target") + .cargo_target_dir() .join(match optimization { BuildOptimization::Release => "release", BuildOptimization::Debug => "debug", diff --git a/lib/bolt/core/src/dep/cargo/cli.rs b/lib/bolt/core/src/dep/cargo/cli.rs index 47f175969..2765018fc 100644 --- a/lib/bolt/core/src/dep/cargo/cli.rs +++ b/lib/bolt/core/src/dep/cargo/cli.rs @@ -88,7 +88,7 @@ pub async fn build<'a, T: AsRef>(ctx: &ProjectContext, opts: BuildOpts<'a, #!/bin/bash set -euf - export CARGO_TARGET_DIR=$(readlink -f ./target) + [ -z "$CARGO_TARGET_DIR" ] && export CARGO_TARGET_DIR=$(readlink -f ./target) # Used for Tokio Console. See https://github.com/tokio-rs/console#using-it export RUSTFLAGS="--cfg tokio_unstable" # Used for debugging diff --git a/lib/bolt/core/src/dep/k8s/gen.rs b/lib/bolt/core/src/dep/k8s/gen.rs index 7acd8ab9e..a623530fa 100644 --- a/lib/bolt/core/src/dep/k8s/gen.rs +++ b/lib/bolt/core/src/dep/k8s/gen.rs @@ -317,7 +317,7 @@ pub async fn gen_svc(exec_ctx: &ExecServiceContext) -> Vec { "IfNotPresent", format!( "{} {}", - Path::new("/rivet-src").join(exec_path).display(), + Path::new("/target").join(exec_path).display(), args.join(" ") ), ), @@ -644,9 +644,9 @@ async fn build_volumes( ExecServiceDriver::LocalBinaryArtifact { .. } => { // Volumes volumes.push(json!({ - "name": "rivet-src", + "name": "target", "hostPath": { - "path": "/rivet-src", + "path": "/target", "type": "Directory" } })); @@ -660,8 +660,8 @@ async fn build_volumes( // Mounts volume_mounts.push(json!({ - "name": "rivet-src", - "mountPath": "/rivet-src", + "name": "target", + "mountPath": "/target", "readOnly": true })); volume_mounts.push(json!({ diff --git a/lib/bolt/core/src/dep/terraform/gen.rs b/lib/bolt/core/src/dep/terraform/gen.rs index 1b4fea67a..87c33a5a4 100644 --- a/lib/bolt/core/src/dep/terraform/gen.rs +++ b/lib/bolt/core/src/dep/terraform/gen.rs @@ -255,6 +255,12 @@ async fn vars(ctx: &ProjectContext) { vars.insert("tunnels".into(), json!(&tunnels)); } + // Rust + vars.insert( + "cargo_target_dir".into(), + json!(ctx.cargo_target_dir().display().to_string()), + ); + // Services { let mut services = HashMap::new(); diff --git a/lib/bolt/core/src/tasks/check.rs b/lib/bolt/core/src/tasks/check.rs index 417b5707c..c6357fbeb 100644 --- a/lib/bolt/core/src/tasks/check.rs +++ b/lib/bolt/core/src/tasks/check.rs @@ -130,7 +130,7 @@ async fn check_svcs( let mut cmd = Command::new("cargo"); cmd.current_dir(path); cmd.env("RUSTFLAGS", "--cfg tokio_unstable"); - cmd.env("CARGO_TARGET_DIR", ctx.path().join("target")); + cmd.env("CARGO_TARGET_DIR", ctx.cargo_target_dir()); cmd.arg("clippy"); // Check tests, which will also check the main module. Using diff --git a/lib/bolt/core/src/tasks/test.rs b/lib/bolt/core/src/tasks/test.rs index d1d1b93d4..579170971 100644 --- a/lib/bolt/core/src/tasks/test.rs +++ b/lib/bolt/core/src/tasks/test.rs @@ -231,9 +231,9 @@ async fn run_test( // Convert path relative to project let relative_path = test_binary .path - .strip_prefix(ctx.path()) + .strip_prefix(ctx.cargo_target_dir()) .context("path not in project")?; - let container_path = Path::new("/rivet-src").join(relative_path); + let container_path = Path::new("/target").join(relative_path); // Build exec ctx let test_id = gen_test_id(); @@ -571,6 +571,10 @@ struct Data { // TODO: This only deletes linodes and firewalls, the ssh key still remains async fn cleanup_servers(ctx: &ProjectContext) -> Result<()> { + if ctx.ns().rivet.dynamic_servers.is_none() { + return Ok(()); + } + eprintln!(); rivet_term::status::progress("Cleaning up servers", ""); diff --git a/lib/bolt/core/src/tasks/up.rs b/lib/bolt/core/src/tasks/up.rs index 5b23cc922..66ef315b6 100644 --- a/lib/bolt/core/src/tasks/up.rs +++ b/lib/bolt/core/src/tasks/up.rs @@ -373,7 +373,7 @@ async fn derive_local_build_driver( RuntimeKind::Rust {} => ExecServiceDriver::LocalBinaryArtifact { // Convert path to be relative to the project root exec_path: exec_path - .strip_prefix(svc_ctx.project().await.path()) + .strip_prefix(svc_ctx.project().await.cargo_target_dir()) .expect("rust binary path not inside of project dir") .to_owned(), args: Vec::new(),