From c148f1923758e34d6c5c33bfb574b48a2c85acd4 Mon Sep 17 00:00:00 2001 From: Bojan Serafimov Date: Thu, 27 Jul 2023 09:50:13 -0400 Subject: [PATCH 1/3] Prewarm compute node --- compute_tools/src/bin/compute_ctl.rs | 3 ++ compute_tools/src/compute.rs | 44 ++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 4953e5f331d0..7d176c51ce6c 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -193,6 +193,9 @@ fn main() -> Result<()> { if !spec_set { // No spec provided, hang waiting for it. info!("no compute spec provided, waiting"); + + compute.prewarm_postgres()?; + let mut state = compute.state.lock().unwrap(); while state.status != ComputeStatus::ConfigurationPending { state = compute.state_changed.wait(state).unwrap(); diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index ceda189364fa..094c42a892cf 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -532,6 +532,50 @@ impl ComputeNode { Ok(()) } + /// Start and stop a postgres process to warm up the VM for startup. + pub fn prewarm_postgres(&self) -> Result<()> { + info!("prewarming"); + + // Create pgdata + let pgdata = "/home/bojan/tmp/prewarm_pgdata"; + create_pgdata(pgdata)?; + + // Run initdb to completion + info!("running initdb"); + let initdb_bin = Path::new(&self.pgbin).parent().unwrap().join("initdb"); + Command::new(initdb_bin) + .args(["-D", pgdata]) + .output() + .expect("cannot start initdb process"); + + // Write conf + use std::io::Write; + let conf_path = Path::new(pgdata).join("postgresql.conf"); + let mut file = std::fs::File::create(conf_path)?; + writeln!(file, "shared_buffers=100MB")?; + writeln!(file, "port=51055")?; // Nobody should be connecting + writeln!(file, "shared_preload_libraries = 'neon'")?; + + // Start postgres + info!("starting postgres"); + let mut pg = Command::new(&self.pgbin) + .args(["-D", pgdata]) + .spawn() + .expect("cannot start postgres process"); + + // Stop it when it's ready + info!("waiting for postgres"); + wait_for_postgres(&mut pg, Path::new(pgdata))?; + pg.kill()?; + info!("sent kill signal"); + pg.wait()?; + info!("done prewarming"); + + // clean up + let _ok = fs::remove_dir_all(pgdata); + Ok(()) + } + /// Start Postgres as a child process and manage DBs/roles. /// After that this will hang waiting on the postmaster process to exit. #[instrument(skip_all)] From e707fbab9edc8a6ef6e170b6554a82c846afa545 Mon Sep 17 00:00:00 2001 From: Bojan Serafimov Date: Thu, 27 Jul 2023 12:54:50 -0400 Subject: [PATCH 2/3] Add todos --- compute_tools/src/compute.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index 094c42a892cf..fdacfa5933b7 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -537,7 +537,7 @@ impl ComputeNode { info!("prewarming"); // Create pgdata - let pgdata = "/home/bojan/tmp/prewarm_pgdata"; + let pgdata = "/home/bojan/tmp/prewarm_pgdata"; // TODO fix this create_pgdata(pgdata)?; // Run initdb to completion @@ -552,7 +552,7 @@ impl ComputeNode { use std::io::Write; let conf_path = Path::new(pgdata).join("postgresql.conf"); let mut file = std::fs::File::create(conf_path)?; - writeln!(file, "shared_buffers=100MB")?; + writeln!(file, "shared_buffers=100MB")?; // TODO is this the default? writeln!(file, "port=51055")?; // Nobody should be connecting writeln!(file, "shared_preload_libraries = 'neon'")?; From bf42201f5a3226ce4542ded567e9df4a4a8c8cee Mon Sep 17 00:00:00 2001 From: Bojan Serafimov Date: Thu, 27 Jul 2023 21:53:16 -0400 Subject: [PATCH 3/3] Fix todos --- compute_tools/src/bin/compute_ctl.rs | 4 ++++ compute_tools/src/compute.rs | 6 +++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 7d176c51ce6c..9d15a203e5b3 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -194,6 +194,10 @@ fn main() -> Result<()> { // No spec provided, hang waiting for it. info!("no compute spec provided, waiting"); + // TODO this can stall startups in the unlikely event that we bind + // this compute node while it's busy prewarming. It's not too + // bad because it's just 100ms and unlikely, but it's an + // avoidable problem. compute.prewarm_postgres()?; let mut state = compute.state.lock().unwrap(); diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index fdacfa5933b7..254d367a7167 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -537,7 +537,7 @@ impl ComputeNode { info!("prewarming"); // Create pgdata - let pgdata = "/home/bojan/tmp/prewarm_pgdata"; // TODO fix this + let pgdata = &format!("{}.warmup", self.pgdata); create_pgdata(pgdata)?; // Run initdb to completion @@ -552,8 +552,8 @@ impl ComputeNode { use std::io::Write; let conf_path = Path::new(pgdata).join("postgresql.conf"); let mut file = std::fs::File::create(conf_path)?; - writeln!(file, "shared_buffers=100MB")?; // TODO is this the default? - writeln!(file, "port=51055")?; // Nobody should be connecting + writeln!(file, "shared_buffers=65536")?; + writeln!(file, "port=51055")?; // Nobody should be connecting writeln!(file, "shared_preload_libraries = 'neon'")?; // Start postgres