chore(backend): cut embed size by ~75% (#544)

packages/backend-embed/Cargo.toml

@@ -18,4 +18,3 @@ sha2 = "0.10.8"
 tempfile = "3.2"
 tokio = { version = "1.40.0", default-features = false, features = ["fs", "rt-multi-thread"] }
 walkdir = "2.5.0"
-

packages/backend-embed/build.rs

@@ -1,5 +1,6 @@
 use anyhow::*;
 use merkle_hash::MerkleTree;
+use std::fs;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
@@ -16,66 +17,74 @@ async fn main() -> Result<()> {
 	script_path.push("build_artifacts.ts");
 
 	let mut backend_path = PathBuf::from(manifest_dir.clone());
-	backend_path.push("..");
+	backend_path.pop();
 	backend_path.push("backend");
 
+	// Copy backend directory to out_dir
+	let out_backend_path = Path::new(&out_dir).join("backend");
+
+	// Remove old dir
+	if out_backend_path.is_dir() {
+		fs::remove_dir_all(&out_backend_path).context("fs::remove_dir_all")?;
+	}
+
+	// TODO: Use native copy dir
+	let status = std::process::Command::new("cp")
+		.arg("-R")
+		.arg(&backend_path)
+		.arg(&out_backend_path)
+		.status()
+		.context("Failed to execute cp command")?;
+	if !status.success() {
+		return Err(anyhow!("cp command failed"));
+	}
+
 	// Install deno
 	let deno_dir = Path::new(&out_dir).join("deno");
 	let deno_exec = rivet_deno_embed::get_executable(&deno_dir).await?;
 
-	// Run script
+	// Run build_artifacts.ts script after copying backend
+	let artifacts_target_dir = out_backend_path.join("artifacts");
 	let status = Command::new(&deno_exec.executable_path)
 		.arg("run")
 		.arg("-A")
 		.arg(&script_path)
+		.current_dir(&out_backend_path)
+		.env(
+			"ARTIFACTS_TARGET_DIR",
+			artifacts_target_dir.to_str().unwrap(),
+		)
 		.status()?;
 	if !status.success() {
 		panic!("build artifacts failed");
 	}
-	println!("cargo:rerun-if-changed={}", script_path.display());
-
-	// Format
-	let status = Command::new(&deno_exec.executable_path)
-		.arg("task")
-		.arg("format")
-		.current_dir(&backend_path)
-		.status()?;
-	if !status.success() {
-		panic!("format failed");
-	}
 
-	// Cache dependencies
+	// Prepare the directory for `include_dir!`
 	let status = Command::new(&deno_exec.executable_path)
 		.arg("task")
-		.arg("cache")
-		.current_dir(&backend_path)
+		.arg("prepare")
+		.current_dir(&out_backend_path)
 		.status()?;
 	if !status.success() {
 		panic!("cache dependencies failed");
 	}
 
-	// Check backend
-	let status = Command::new(&deno_exec.executable_path)
-		.arg("task")
-		.arg("check")
-		.current_dir(&backend_path)
-		.status()?;
-	if !status.success() {
-		panic!("check files failed");
-	}
+	// TODO: This doesn't work
+	// Removes files that are not cross-platform & deletes
+	// broken symlinks.
+	// strip_cross_platform(&out_backend_path)?;
 
-	// Hash backend
-	let mut backend_path = PathBuf::from(manifest_dir);
-	backend_path.pop();
-	backend_path.push("backend");
+	println!("cargo:rerun-if-changed={}", script_path.display());
+	println!("cargo:rerun-if-changed={}", backend_path.display());
+	println!(
+		"cargo:rustc-env=BACKEND_PATH={}",
+		out_backend_path.display()
+	);
 	println!(
 		"cargo:rustc-env=BACKEND_HASH={}",
-		hash_directory(&backend_path)?
+		hash_directory(&out_backend_path)?
 	);
 
-	// TODO: Add back OGBEE-129
-	// println!("cargo:rerun-if-changed={}", backend_path.display());
-
 	Ok(())
 }
 
@@ -91,3 +100,56 @@ fn hash_directory<P: AsRef<Path>>(path: P) -> Result<String> {
 		.join("");
 	Ok(hash)
 }
+
+// fn strip_cross_platform(path: &Path) -> Result<()> {
+// 	// Remove directories starting with "@esbuild+"
+// 	let esbuild_path = path.join("node_modules").join(".deno");
+// 	let output = Command::new("find")
+// 		.arg(&esbuild_path)
+// 		.arg("-type")
+// 		.arg("d")
+// 		.arg("-name")
+// 		.arg("@esbuild+*")
+// 		.arg("-exec")
+// 		.arg("rm")
+// 		.arg("-rf")
+// 		.arg("{}")
+// 		.arg("+")
+// 		.output()
+// 		.context("Failed to execute 'find' command to remove @esbuild+ directories")?;
+//
+// 	if !output.status.success() {
+// 		return Err(anyhow!(
+// 			"Failed to remove @esbuild+ directories. Path: {}, Status: {}, Stdout: {}, Stderr: {}",
+// 			esbuild_path.display(),
+// 			output.status,
+// 			String::from_utf8_lossy(&output.stdout),
+// 			String::from_utf8_lossy(&output.stderr)
+// 		));
+// 	}
+//
+// 	// Remove broken symlinks
+// 	let output = Command::new("find")
+// 		.arg(path)
+// 		.arg("-type")
+// 		.arg("l")
+// 		.arg("-exec")
+// 		.arg("sh")
+// 		.arg("-c")
+// 		.arg("for x; do [ -e \"$x\" ] || rm \"$x\"; done")
+// 		.arg("{}")
+// 		.arg("+")
+// 		.output()
+// 		.context("Failed to execute 'find' command to remove broken symlinks")?;
+//
+// 	if !output.status.success() {
+// 		return Err(anyhow!(
+// 			"Failed to remove broken symlinks. Status: {}, Stdout: {}, Stderr: {}",
+// 			output.status,
+// 			String::from_utf8_lossy(&output.stdout),
+// 			String::from_utf8_lossy(&output.stderr)
+// 		));
+// 	}
+//
+// 	Ok(())
+// }

packages/backend-embed/src/lib.rs

@@ -3,7 +3,7 @@ use include_dir::{include_dir, Dir};
 use std::path::PathBuf;
 use tokio::fs;
 
-const BACKEND_DIR: Dir = include_dir!("$CARGO_MANIFEST_DIR/../backend");
+const BACKEND_DIR: Dir = include_dir!("$BACKEND_PATH");
 const BACKEND_HASH: &'static str = env!("BACKEND_HASH");
 
 /// Return a path for the backend. If one does not exist, the backend dir will automatically be

packages/backend/deno.jsonc

@@ -6,8 +6,20 @@
 
 		"check": "deno check cli/**/*.ts toolchain/**/*.ts runtime/**/*.ts",
 
-		// Remove old vendored files in order to ensure a consistent cache
-		"cache": "rm -rf vendor node_modules && deno cache toolchain/**/*.ts runtime/**/*.ts",
+		// Remove old vendored files in order to ensure a consistent cache.
+		//
+		// The deno.lock needs to be cleared to prevent from caching unused
+		// dependencies.
+		//
+    // strip_cross_platform removes files that are not cross-platform & deletes
+    // broken symlinks.
+		"cache": "deno task cache:purge && deno task cache:download",
+		"cache:purge": "rm -rf deno.lock vendor node_modules",
+		"cache:download": "deno cache cli/tasks/**/*.ts",
+
+		// Cache must be the last step in order to prevent unwanted extra cached
+		// files
+		"prepare": "deno task format && deno task check && deno task cache",
 
 		"lint": "deno lint cli/**/*.ts toolchain/**/*.ts runtime/**/*.ts",
 		"lint:fix": "deno lint cli/**/*.ts toolchain/**/*.ts runtime/**/*.ts",