Auto merge of rust-lang#98373 - joshtriplett:bootstrap-locking, r=jyn514

Move locking from bootstrap.py to rust bootstrap, using fd-lock

Helps with rust-lang#94829.

src/bootstrap/Cargo.lock

@@ -47,6 +47,7 @@ version = "0.0.0"
 dependencies = [
  "cc",
  "cmake",
+ "fd-lock",
  "filetime",
  "getopts",
  "hex",
@@ -201,6 +202,38 @@ version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
 
+[[package]]
+name = "errno"
+version = "0.2.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1"
+dependencies = [
+ "errno-dragonfly",
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "errno-dragonfly"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf"
+dependencies = [
+ "cc",
+ "libc",
+]
+
+[[package]]
+name = "fd-lock"
+version = "3.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e11dcc7e4d79a8c89b9ab4c6f5c30b1fc4a83c420792da3542fd31179ed5f517"
+dependencies = [
+ "cfg-if",
+ "rustix",
+ "windows-sys",
+]
+
 [[package]]
 name = "filetime"
 version = "0.2.16"
@@ -284,6 +317,12 @@ dependencies = [
  "winapi-util",
 ]
 
+[[package]]
+name = "io-lifetimes"
+version = "0.7.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24c3f4eff5495aee4c0399d7b6a0dc2b6e81be84242ffbfcf253ebacccc1d0cb"
+
 [[package]]
 name = "itoa"
 version = "1.0.2"
@@ -302,6 +341,12 @@ version = "0.2.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "349d5a591cd28b49e1d1037471617a32ddcda5731b99419008085f72d5a53836"
 
+[[package]]
+name = "linux-raw-sys"
+version = "0.0.46"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d"
+
 [[package]]
 name = "log"
 version = "0.4.17"
@@ -473,6 +518,20 @@ version = "0.6.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64"
 
+[[package]]
+name = "rustix"
+version = "0.35.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ef258c11e17f5c01979a10543a30a4e12faef6aab217a74266e747eefa3aed88"
+dependencies = [
+ "bitflags",
+ "errno",
+ "io-lifetimes",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys",
+]
+
 [[package]]
 name = "ryu"
 version = "1.0.10"
@@ -657,6 +716,49 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
+[[package]]
+name = "windows-sys"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea04155a16a59f9eab786fe12a4a450e75cdb175f9e0d80da1e17db09f55b8d2"
+dependencies = [
+ "windows_aarch64_msvc",
+ "windows_i686_gnu",
+ "windows_i686_msvc",
+ "windows_x86_64_gnu",
+ "windows_x86_64_msvc",
+]
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9bb8c3fd39ade2d67e9874ac4f3db21f0d710bee00fe7cab16949ec184eeaa47"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "180e6ccf01daf4c426b846dfc66db1fc518f074baa793aa7d9b9aaeffad6a3b6"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e2e7917148b2812d1eeafaeb22a97e4813dfa60a3f8f78ebe204bcc88f12f024"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4dcd171b8776c41b97521e5da127a2d86ad280114807d0b2ab1e462bc764d9e1"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.36.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c811ca4a8c853ef420abd8592ba53ddbbac90410fab6903b3e79972a631f7680"
+
 [[package]]
 name = "xattr"
 version = "0.2.3"

src/bootstrap/Cargo.toml

@@ -36,6 +36,7 @@ test = false
 
 [dependencies]
 cmake = "0.1.38"
+fd-lock = "3.0.6"
 filetime = "0.2"
 num_cpus = "1.0"
 getopts = "0.2.19"

src/bootstrap/bin/main.rs

@@ -7,12 +7,28 @@
 
 use std::env;
 
-use bootstrap::{Build, Config, Subcommand, VERSION};
+use bootstrap::{t, Build, Config, Subcommand, VERSION};
 
 fn main() {
     let args = env::args().skip(1).collect::<Vec<_>>();
     let config = Config::parse(&args);
 
+    let mut build_lock;
+    let _build_lock_guard;
+    if cfg!(any(unix, windows)) {
+        build_lock = fd_lock::RwLock::new(t!(std::fs::File::create(config.out.join("lock"))));
+        _build_lock_guard = match build_lock.try_write() {
+            Ok(lock) => lock,
+            err => {
+                println!("warning: build directory locked, waiting for lock");
+                drop(err);
+                t!(build_lock.write())
+            }
+        };
+    } else {
+        println!("warning: file locking not supported for target, not locking build directory");
+    }
+
     // check_version warnings are not printed during setup
     let changelog_suggestion =
         if matches!(config.cmd, Subcommand::Setup { .. }) { None } else { check_version(&config) };

src/bootstrap/bootstrap.py

@@ -15,44 +15,6 @@
 
 from time import time, sleep
 
-# Acquire a lock on the build directory to make sure that
-# we don't cause a race condition while building
-# Lock is created in `build_dir/lock.db`
-def acquire_lock(build_dir):
-    try:
-        import sqlite3
-
-        path = os.path.join(build_dir, "lock.db")
-        try:
-            con = sqlite3.Connection(path, timeout=0)
-            curs = con.cursor()
-            curs.execute("BEGIN EXCLUSIVE")
-            # The lock is released when the cursor is dropped
-            return curs
-        # If the database is busy then lock has already been acquired
-        # so we wait for the lock.
-        # We retry every quarter second so that execution is passed back to python
-        # so that it can handle signals
-        except sqlite3.OperationalError:
-            del con
-            del curs
-            print("Waiting for lock on build directory")
-            con = sqlite3.Connection(path, timeout=0.25)
-            curs = con.cursor()
-            while True:
-                try:
-                    curs.execute("BEGIN EXCLUSIVE")
-                    break
-                except sqlite3.OperationalError:
-                    pass
-                sleep(0.25)
-            return curs
-    except ImportError:
-        print("warning: sqlite3 not available in python, skipping build directory lock")
-        print("please file an issue on rust-lang/rust")
-        print("this is not a problem for non-concurrent x.py invocations")
-        return None
-
 def support_xz():
     try:
         with tempfile.NamedTemporaryFile(delete=False) as temp_file:
@@ -928,11 +890,8 @@ def bootstrap(help_triggered):
 
     build.build = args.build or build.build_triple()
 
-    # Acquire the lock before doing any build actions
-    # The lock is released when `lock` is dropped
     if not os.path.exists(build.build_dir):
         os.makedirs(build.build_dir)
-    lock = acquire_lock(build.build_dir)
 
     # Fetch/build the bootstrap
     build.download_toolchain()

src/bootstrap/lib.rs

@@ -118,8 +118,7 @@ use once_cell::sync::OnceCell;
 use crate::builder::Kind;
 use crate::config::{LlvmLibunwind, TargetSelection};
 use crate::util::{
-    check_run, exe, libdir, mtime, output, run, run_suppressed, t, try_run, try_run_suppressed,
-    CiEnv,
+    check_run, exe, libdir, mtime, output, run, run_suppressed, try_run, try_run_suppressed, CiEnv,
 };
 
 mod builder;

src/bootstrap/util.rs

@@ -22,6 +22,7 @@ use crate::config::{Config, TargetSelection};
 ///
 /// This is currently used judiciously throughout the build system rather than
 /// using a `Result` with `try!`, but this may change one day...
+#[macro_export]
 macro_rules! t {
     ($e:expr) => {
         match $e {
@@ -37,7 +38,7 @@ macro_rules! t {
         }
     };
 }
-pub(crate) use t;
+pub use t;
 
 /// Given an executable called `name`, return the filename for the
 /// executable for a particular target.