From f8c4ad3b18ce71474ed60986d600fa59780a4b8b Mon Sep 17 00:00:00 2001 From: SparrowLii Date: Sun, 23 Apr 2023 10:38:16 +0800 Subject: [PATCH 01/11] use dyn send/sync data structures --- compiler/rustc_data_structures/src/sync.rs | 653 +++++++++++++++++---- 1 file changed, 554 insertions(+), 99 deletions(-) diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs index 6c3197d8ec2c5..fe48178361020 100644 --- a/compiler/rustc_data_structures/src/sync.rs +++ b/compiler/rustc_data_structures/src/sync.rs @@ -40,8 +40,12 @@ //! [^2] `MTLockRef` is a typedef. pub use crate::marker::*; +use std::cell::{Cell, UnsafeCell}; use std::collections::HashMap; +use std::fmt::{Debug, Formatter}; use std::hash::{BuildHasher, Hash}; +use std::intrinsics::likely; +use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe}; @@ -54,6 +58,9 @@ pub use std::sync::atomic::Ordering::SeqCst; pub use vec::{AppendOnlyIndexVec, AppendOnlyVec}; mod vec; +use parking_lot::lock_api::RawMutex as _; +use parking_lot::lock_api::RawRwLock as _; +use parking_lot::{RawMutex, RawRwLock}; mod mode { use super::Ordering; @@ -246,20 +253,9 @@ cfg_if! { pub use std::rc::Rc as Lrc; pub use std::rc::Weak as Weak; - pub use std::cell::Ref as ReadGuard; - pub use std::cell::Ref as MappedReadGuard; - pub use std::cell::RefMut as WriteGuard; - pub use std::cell::RefMut as MappedWriteGuard; - pub use std::cell::RefMut as LockGuard; - pub use std::cell::RefMut as MappedLockGuard; pub use std::cell::OnceCell; - use std::cell::RefCell as InnerRwLock; - use std::cell::RefCell as InnerLock; - - use std::cell::Cell; - pub type MTLockRef<'a, T> = &'a mut MTLock; #[derive(Debug, Default)] @@ -303,14 +299,6 @@ cfg_if! { pub use std::marker::Send as Send; pub use std::marker::Sync as Sync; - pub use parking_lot::RwLockReadGuard as ReadGuard; - pub use parking_lot::MappedRwLockReadGuard as MappedReadGuard; - pub use parking_lot::RwLockWriteGuard as WriteGuard; - pub use parking_lot::MappedRwLockWriteGuard as MappedWriteGuard; - - pub use parking_lot::MutexGuard as LockGuard; - pub use parking_lot::MappedMutexGuard as MappedLockGuard; - pub use std::sync::OnceLock as OnceCell; pub use std::sync::atomic::{AtomicBool, AtomicUsize, AtomicU32, AtomicU64}; @@ -350,8 +338,21 @@ cfg_if! { } } - use parking_lot::Mutex as InnerLock; - use parking_lot::RwLock as InnerRwLock; + #[inline] + pub fn join(oper_a: A, oper_b: B) -> (RA, RB) + where + A: FnOnce() -> RA + DynSend, + B: FnOnce() -> RB + DynSend, + { + if mode::active() { + let oper_a = FromDyn::from(oper_a); + let oper_b = FromDyn::from(oper_b); + let (a, b) = rayon::join(move || FromDyn::from(oper_a.into_inner()()), move || FromDyn::from(oper_b.into_inner()())); + (a.into_inner(), b.into_inner()) + } else { + (oper_a(), oper_b()) + } + } use std::thread; @@ -385,11 +386,19 @@ cfg_if! { /// the current thread. Use that for the longest running block. #[macro_export] macro_rules! parallel { + (impl $fblock:tt [$($c:tt,)*] [$block:tt $(, $rest:tt)*]) => { (impl $fblock:block [$($c:expr,)*] [$block:expr $(, $rest:expr)*]) => { parallel!(impl $fblock [$block, $($c,)*] [$($rest),*]) }; + (impl $fblock:tt [$($blocks:tt,)*] []) => { (impl $fblock:block [$($blocks:expr,)*] []) => { ::rustc_data_structures::sync::scope(|s| { + + + + + + $(let block = rustc_data_structures::sync::FromDyn::from(|| $blocks); s.spawn(move |_| block.into_inner()());)* (|| $fblock)(); @@ -413,6 +422,7 @@ cfg_if! { } } $( + s.spawn(|_| $blocks); if let Err(p) = ::std::panic::catch_unwind( ::std::panic::AssertUnwindSafe(|| $blocks) ) { @@ -421,11 +431,19 @@ cfg_if! { } } )* + $fblock; + }) if let Some(panic) = panic { ::std::panic::resume_unwind(panic); } } }; + ($fblock:tt, $($blocks:tt),*) => { + // Reverse the order of the later blocks since Rayon executes them in reverse order + // when using a single thread. This ensures the execution order matches that + // of a single threaded rustc + parallel!(impl $fblock [] [$($blocks),*]); + }; } use rayon::iter::{FromParallelIterator, IntoParallelIterator, ParallelIterator}; @@ -515,9 +533,29 @@ cfg_if! { } } - /// This makes locks panic if they are already held. - /// It is only useful when you are running in a single thread - const ERROR_CHECKING: bool = false; +pub unsafe trait DynSend {} +pub unsafe trait DynSync {} + +unsafe impl DynSend for T where T: Send {} +unsafe impl DynSync for T where T: Sync {} + +#[derive(Copy, Clone)] +pub struct FromDyn(T); + +impl FromDyn { + #[inline(always)] + pub fn from(val: T) -> Self { + // Check that `sync::active()` is true on creation so we can + // implement `Send` and `Sync` for this structure when `T` + // implements `DynSend` and `DynSync` respectively. + #[cfg(parallel_compiler)] + assert!(mode::active()); + FromDyn(val) + } + + #[inline(always)] + pub fn into_inner(self) -> T { + self.0 } } @@ -537,59 +575,135 @@ impl HashMapExt for HashMap } } -#[derive(Debug)] -pub struct Lock(InnerLock); +/// This makes locks panic if they are already held. +/// It is only useful when you are running in a single thread +// const ERROR_CHECKING: bool = false; + +pub struct Lock { + single_thread: bool, + pub(crate) data: UnsafeCell, + pub(crate) borrow: Cell, + mutex: RawMutex, +} + +impl Debug for Lock { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self.try_lock() { + Some(guard) => f.debug_struct("Lock").field("data", guard.deref()).finish(), + None => { + struct LockedPlaceholder; + impl Debug for LockedPlaceholder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str("") + } + } + + f.debug_struct("Lock").field("data", &LockedPlaceholder).finish() + } + } + } +} impl Lock { - #[inline(always)] - pub fn new(inner: T) -> Self { - Lock(InnerLock::new(inner)) + #[inline] + pub fn new(val: T) -> Self { + Lock { + single_thread: !active(), + data: UnsafeCell::new(val), + borrow: Cell::new(false), + mutex: RawMutex::INIT, + } } - #[inline(always)] + #[inline] pub fn into_inner(self) -> T { - self.0.into_inner() + self.data.into_inner() } - #[inline(always)] + #[inline] pub fn get_mut(&mut self) -> &mut T { - self.0.get_mut() + self.data.get_mut() } - #[cfg(parallel_compiler)] - #[inline(always)] + #[inline] pub fn try_lock(&self) -> Option> { - self.0.try_lock() + // SAFETY: the `&mut T` is accessible as long as self exists. + if likely(self.single_thread) { + if self.borrow.get() { + None + } else { + self.borrow.set(true); + Some(LockGuard { lock: &self, marker: PhantomData }) + } + } else { + if !self.mutex.try_lock() { + None + } else { + Some(LockGuard { lock: &self, marker: PhantomData }) + } + } } - #[cfg(not(parallel_compiler))] - #[inline(always)] - pub fn try_lock(&self) -> Option> { - self.0.try_borrow_mut().ok() + #[inline(never)] + fn lock_raw(&self) { + if likely(self.single_thread) { + assert!(!self.borrow.replace(true)); + } else { + self.mutex.lock(); + } } - #[cfg(parallel_compiler)] #[inline(always)] #[track_caller] pub fn lock(&self) -> LockGuard<'_, T> { - if ERROR_CHECKING { - self.0.try_lock().expect("lock was already held") - } else { - self.0.lock() + self.lock_raw(); + LockGuard { lock: &self, marker: PhantomData } + } + + #[inline(never)] + pub(crate) fn with_mt_lock R, R>(&self, f: F) -> R { + unsafe { + self.mutex.lock(); + let r = f(&mut *self.data.get()); + self.mutex.unlock(); + r } } - #[cfg(not(parallel_compiler))] #[inline(always)] #[track_caller] - pub fn lock(&self) -> LockGuard<'_, T> { - self.0.borrow_mut() + pub fn with_lock R, R>(&self, f: F) -> R { + if likely(self.single_thread) { + assert!(!self.borrow.replace(true)); + let r = unsafe { f(&mut *self.data.get()) }; + self.borrow.set(false); + r + } else { + self.with_mt_lock(f) + } + } + + #[inline(never)] + fn with_mt_borrow R, R>(&self, f: F) -> R { + unsafe { + self.mutex.lock(); + let r = f(&*self.data.get()); + self.mutex.unlock(); + r + } } #[inline(always)] #[track_caller] - pub fn with_lock R, R>(&self, f: F) -> R { - f(&mut *self.lock()) + pub fn with_borrow R, R>(&self, f: F) -> R { + if likely(self.single_thread) { + assert!(!self.borrow.replace(true)); + let r = unsafe { f(&*self.data.get()) }; + self.borrow.set(false); + r + } else { + self.with_mt_borrow(f) + } } #[inline(always)] @@ -612,81 +726,379 @@ impl Default for Lock { } } -#[derive(Debug, Default)] -pub struct RwLock(InnerRwLock); +// Just for speed test +unsafe impl std::marker::Send for Lock {} +unsafe impl std::marker::Sync for Lock {} + +pub struct LockGuard<'a, T> { + lock: &'a Lock, + marker: PhantomData<&'a mut T>, +} + +impl Deref for LockGuard<'_, T> { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &T { + unsafe { &*self.lock.data.get() } + } +} + +impl DerefMut for LockGuard<'_, T> { + #[inline(always)] + fn deref_mut(&mut self) -> &mut T { + unsafe { &mut *self.lock.data.get() } + } +} + +#[inline(never)] +fn unlock_mt(guard: &mut LockGuard<'_, T>) { + unsafe { guard.lock.mutex.unlock() } +} + +impl<'a, T> Drop for LockGuard<'a, T> { + #[inline] + fn drop(&mut self) { + if likely(self.lock.single_thread) { + debug_assert!(self.lock.borrow.get()); + self.lock.borrow.set(false); + } else { + unlock_mt(self) + } + } +} + +pub struct MappedReadGuard<'a, T: ?Sized> { + raw: &'a RwLockRaw, + data: *const T, + marker: PhantomData<&'a T>, +} + +unsafe impl std::marker::Send for MappedReadGuard<'_, T> {} +unsafe impl std::marker::Sync for MappedReadGuard<'_, T> {} + +impl<'a, T: 'a + ?Sized> MappedReadGuard<'a, T> { + #[inline] + pub fn map(s: Self, f: F) -> MappedReadGuard<'a, U> + where + F: FnOnce(&T) -> &U, + { + let raw = s.raw; + let data = f(unsafe { &*s.data }); + std::mem::forget(s); + MappedReadGuard { raw, data, marker: PhantomData } + } +} + +impl<'a, T: 'a + ?Sized> Deref for MappedReadGuard<'a, T> { + type Target = T; + #[inline] + fn deref(&self) -> &T { + unsafe { &*self.data } + } +} + +impl<'a, T: 'a + ?Sized> Drop for MappedReadGuard<'a, T> { + #[inline] + fn drop(&mut self) { + if likely(self.raw.single_thread) { + let i = self.raw.borrow.get(); + debug_assert!(i > 0); + self.raw.borrow.set(i - 1); + } else { + // Safety: An RwLockReadGuard always holds a shared lock. + unsafe { + self.raw.raw.unlock_shared(); + } + } + } +} + +pub struct MappedWriteGuard<'a, T: ?Sized> { + raw: &'a RwLockRaw, + data: *mut T, + marker: PhantomData<&'a mut T>, +} + +unsafe impl std::marker::Send for MappedWriteGuard<'_, T> {} + +impl<'a, T: 'a + ?Sized> MappedWriteGuard<'a, T> { + #[inline] + pub fn map(s: Self, f: F) -> MappedWriteGuard<'a, U> + where + F: FnOnce(&mut T) -> &mut U, + { + let raw = s.raw; + let data = f(unsafe { &mut *s.data }); + std::mem::forget(s); + MappedWriteGuard { raw, data, marker: PhantomData } + } +} + +impl<'a, T: 'a + ?Sized> Deref for MappedWriteGuard<'a, T> { + type Target = T; + #[inline] + fn deref(&self) -> &T { + unsafe { &*self.data } + } +} + +impl<'a, T: 'a + ?Sized> DerefMut for MappedWriteGuard<'a, T> { + #[inline] + fn deref_mut(&mut self) -> &mut T { + unsafe { &mut *self.data } + } +} + +impl<'a, T: 'a + ?Sized> Drop for MappedWriteGuard<'a, T> { + #[inline] + fn drop(&mut self) { + if likely(self.raw.single_thread) { + assert_eq!(self.raw.borrow.replace(0), -1); + } else { + // Safety: An RwLockReadGuard always holds a shared lock. + unsafe { + self.raw.raw.unlock_exclusive(); + } + } + } +} + +pub struct ReadGuard<'a, T> { + rwlock: &'a RwLock, + marker: PhantomData<&'a T>, +} + +impl<'a, T: 'a> ReadGuard<'a, T> { + pub fn map(s: Self, f: F) -> MappedReadGuard<'a, U> + where + F: FnOnce(&T) -> &U, + { + let raw = &s.rwlock.raw; + let data = f(unsafe { &*s.rwlock.data.get() }); + std::mem::forget(s); + MappedReadGuard { raw, data, marker: PhantomData } + } +} + +impl<'a, T: 'a> Deref for ReadGuard<'a, T> { + type Target = T; + #[inline] + fn deref(&self) -> &T { + unsafe { &*self.rwlock.data.get() } + } +} + +impl<'a, T: 'a> Drop for ReadGuard<'a, T> { + #[inline] + fn drop(&mut self) { + if likely(self.rwlock.raw.single_thread) { + let i = self.rwlock.raw.borrow.get(); + debug_assert!(i > 0); + self.rwlock.raw.borrow.set(i - 1); + } else { + // Safety: An RwLockReadGuard always holds a shared lock. + unsafe { + self.rwlock.raw.raw.unlock_shared(); + } + } + } +} + +pub struct WriteGuard<'a, T> { + rwlock: &'a RwLock, + marker: PhantomData<&'a mut T>, +} + +impl<'a, T: 'a> WriteGuard<'a, T> { + pub fn map(s: Self, f: F) -> MappedWriteGuard<'a, U> + where + F: FnOnce(&mut T) -> &mut U, + { + let raw = &s.rwlock.raw; + let data = f(unsafe { &mut *s.rwlock.data.get() }); + std::mem::forget(s); + MappedWriteGuard { raw, data, marker: PhantomData } + } +} + +impl<'a, T: 'a> Deref for WriteGuard<'a, T> { + type Target = T; + #[inline] + fn deref(&self) -> &T { + unsafe { &*self.rwlock.data.get() } + } +} + +impl<'a, T: 'a> DerefMut for WriteGuard<'a, T> { + #[inline] + fn deref_mut(&mut self) -> &mut T { + unsafe { &mut *self.rwlock.data.get() } + } +} + +impl<'a, T: 'a> Drop for WriteGuard<'a, T> { + #[inline] + fn drop(&mut self) { + if likely(self.rwlock.raw.single_thread) { + assert_eq!(self.rwlock.raw.borrow.replace(0), -1); + } else { + // Safety: An RwLockWriteGuard always holds an exclusive lock. + unsafe { + self.rwlock.raw.raw.unlock_exclusive(); + } + } + } +} + +struct RwLockRaw { + single_thread: bool, + borrow: Cell, + raw: RawRwLock, +} + +pub struct RwLock { + raw: RwLockRaw, + data: UnsafeCell, +} + +impl Debug for RwLock { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Lock").field("data", self.read().deref()).finish() + } +} + +impl Default for RwLock { + fn default() -> Self { + RwLock { + raw: RwLockRaw { single_thread: !active(), borrow: Cell::new(0), raw: RawRwLock::INIT }, + + data: UnsafeCell::new(T::default()), + } + } +} impl RwLock { #[inline(always)] pub fn new(inner: T) -> Self { - RwLock(InnerRwLock::new(inner)) + RwLock { + raw: RwLockRaw { single_thread: !active(), borrow: Cell::new(0), raw: RawRwLock::INIT }, + + data: UnsafeCell::new(inner), + } } #[inline(always)] pub fn into_inner(self) -> T { - self.0.into_inner() + self.data.into_inner() } #[inline(always)] pub fn get_mut(&mut self) -> &mut T { - self.0.get_mut() + self.data.get_mut() } - #[cfg(not(parallel_compiler))] - #[inline(always)] - #[track_caller] - pub fn read(&self) -> ReadGuard<'_, T> { - self.0.borrow() + #[inline(never)] + fn mt_read(&self) -> ReadGuard<'_, T> { + self.raw.raw.lock_shared(); + ReadGuard { rwlock: self, marker: PhantomData } } - #[cfg(parallel_compiler)] #[inline(always)] pub fn read(&self) -> ReadGuard<'_, T> { - if ERROR_CHECKING { - self.0.try_read().expect("lock was already held") + if likely(self.raw.single_thread) { + let b = self.raw.borrow.get(); + assert!(b >= 0); + self.raw.borrow.set(b + 1); + ReadGuard { rwlock: self, marker: PhantomData } } else { - self.0.read() + self.mt_read() } } + #[inline(never)] + fn with_mt_read_lock R, R>(&self, f: F) -> R { + self.raw.raw.lock_shared(); + let r = unsafe { f(&*self.data.get()) }; + unsafe { + self.raw.raw.unlock_shared(); + } + r + } + #[inline(always)] #[track_caller] pub fn with_read_lock R, R>(&self, f: F) -> R { - f(&*self.read()) + if likely(self.raw.single_thread) { + let b = self.raw.borrow.get(); + assert!(b >= 0); + self.raw.borrow.set(b + 1); + let r = unsafe { f(&*self.data.get()) }; + self.raw.borrow.set(b); + r + } else { + self.with_mt_read_lock(f) + } } - #[cfg(not(parallel_compiler))] #[inline(always)] pub fn try_write(&self) -> Result, ()> { - self.0.try_borrow_mut().map_err(|_| ()) + if likely(self.raw.single_thread) { + let b = self.raw.borrow.get(); + if b != 0 { + Err(()) + } else { + self.raw.borrow.set(-1); + Ok(WriteGuard { rwlock: self, marker: PhantomData }) + } + } else { + if self.raw.raw.try_lock_exclusive() { + Ok(WriteGuard { rwlock: self, marker: PhantomData }) + } else { + Err(()) + } + } } - #[cfg(parallel_compiler)] - #[inline(always)] - pub fn try_write(&self) -> Result, ()> { - self.0.try_write().ok_or(()) + #[inline(never)] + fn mt_write(&self) -> WriteGuard<'_, T> { + self.raw.raw.lock_exclusive(); + WriteGuard { rwlock: self, marker: PhantomData } } - #[cfg(not(parallel_compiler))] #[inline(always)] - #[track_caller] pub fn write(&self) -> WriteGuard<'_, T> { - self.0.borrow_mut() + if likely(self.raw.single_thread) { + assert_eq!(self.raw.borrow.replace(-1), 0); + WriteGuard { rwlock: self, marker: PhantomData } + } else { + self.mt_write() + } } - #[cfg(parallel_compiler)] - #[inline(always)] - pub fn write(&self) -> WriteGuard<'_, T> { - if ERROR_CHECKING { - self.0.try_write().expect("lock was already held") - } else { - self.0.write() + #[inline(never)] + pub fn with_mt_write_lock R, R>(&self, f: F) -> R { + self.raw.raw.lock_exclusive(); + unsafe { + let r = f(&mut *self.data.get()); + self.raw.raw.unlock_exclusive(); + r } } #[inline(always)] #[track_caller] pub fn with_write_lock R, R>(&self, f: F) -> R { - f(&mut *self.write()) + if likely(self.raw.single_thread) { + let b = self.raw.borrow.get(); + assert!(b >= 0); + self.raw.borrow.set(b + 1); + let r = unsafe { f(&mut *self.data.get()) }; + self.raw.borrow.set(b); + r + } else { + self.with_mt_write_lock(f) + } } #[inline(always)] @@ -701,13 +1113,6 @@ impl RwLock { self.write() } - #[cfg(not(parallel_compiler))] - #[inline(always)] - pub fn leak(&self) -> &T { - ReadGuard::leak(self.read()) - } - - #[cfg(parallel_compiler)] #[inline(always)] pub fn leak(&self) -> &T { let guard = self.read(); @@ -717,6 +1122,10 @@ impl RwLock { } } +// just for speed test +unsafe impl std::marker::Send for RwLock {} +unsafe impl std::marker::Sync for RwLock {} + // FIXME: Probably a bad idea impl Clone for RwLock { #[inline] @@ -725,34 +1134,80 @@ impl Clone for RwLock { } } +#[derive(Debug)] +pub struct WorkerLocal { + single_thread: bool, + inner: Option, + mt_inner: Option>, +} + +impl WorkerLocal { + /// Creates a new worker local where the `initial` closure computes the + /// value this worker local should take for each thread in the thread pool. + #[inline] + pub fn new T>(mut f: F) -> WorkerLocal { + if !active() { + WorkerLocal { single_thread: true, inner: Some(f(0)), mt_inner: None } + } else { + WorkerLocal { + single_thread: false, + inner: None, + mt_inner: Some(rayon_core::WorkerLocal::new(f)), + } + } + } + + /// Returns the worker-local value for each thread + #[inline] + pub fn into_inner(self) -> Vec { + if self.single_thread { + vec![self.inner.unwrap()] + } else { + self.mt_inner.unwrap().into_inner() + } + } +} + +impl Deref for WorkerLocal { + type Target = T; + + #[inline(always)] + fn deref(&self) -> &T { + if self.single_thread { + self.inner.as_ref().unwrap() + } else { + self.mt_inner.as_ref().unwrap().deref() + } + } +} + +// Just for speed test +unsafe impl std::marker::Sync for WorkerLocal {} + +use std::thread; + /// A type which only allows its inner value to be used in one thread. /// It will panic if it is used on multiple threads. #[derive(Debug)] pub struct OneThread { - #[cfg(parallel_compiler)] + single_thread: bool, thread: thread::ThreadId, inner: T, } -#[cfg(parallel_compiler)] +// just for speed test now unsafe impl std::marker::Sync for OneThread {} -#[cfg(parallel_compiler)] unsafe impl std::marker::Send for OneThread {} impl OneThread { #[inline(always)] fn check(&self) { - #[cfg(parallel_compiler)] - assert_eq!(thread::current().id(), self.thread); + assert!(self.single_thread || thread::current().id() == self.thread); } #[inline(always)] pub fn new(inner: T) -> Self { - OneThread { - #[cfg(parallel_compiler)] - thread: thread::current().id(), - inner, - } + OneThread { single_thread: !active(), thread: thread::current().id(), inner } } #[inline(always)] From 8fbb9113bf6ae0040e4fcc69361b5c830ef49f40 Mon Sep 17 00:00:00 2001 From: SparrowLii Date: Sun, 23 Apr 2023 10:39:07 +0800 Subject: [PATCH 02/11] let SHARD = 1 --- compiler/rustc_data_structures/Cargo.toml | 4 +- compiler/rustc_data_structures/src/lib.rs | 1 + compiler/rustc_data_structures/src/sharded.rs | 93 ++++++++++--------- 3 files changed, 53 insertions(+), 45 deletions(-) diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml index 78f73d193e380..b92d57ea4ad2d 100644 --- a/compiler/rustc_data_structures/Cargo.toml +++ b/compiler/rustc_data_structures/Cargo.toml @@ -14,7 +14,7 @@ indexmap = { version = "1.9.3" } jobserver_crate = { version = "0.1.13", package = "jobserver" } libc = "0.2" measureme = "10.0.0" -rustc-rayon-core = { version = "0.5.0", optional = true } +rustc-rayon-core = { version = "0.5.0" } rustc-rayon = { version = "0.5.0", optional = true } rustc_arena = { path = "../rustc_arena" } rustc_graphviz = { path = "../rustc_graphviz" } @@ -51,4 +51,4 @@ features = [ memmap2 = "0.2.1" [features] -rustc_use_parallel_compiler = ["indexmap/rustc-rayon", "rustc-rayon", "rustc-rayon-core"] +rustc_use_parallel_compiler = ["indexmap/rustc-rayon", "rustc-rayon"] \ No newline at end of file diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs index 5b9b0e106d254..fdd09d59a76c2 100644 --- a/compiler/rustc_data_structures/src/lib.rs +++ b/compiler/rustc_data_structures/src/lib.rs @@ -33,6 +33,7 @@ #![feature(strict_provenance)] #![feature(ptr_alignment_type)] #![feature(macro_metavar_expr)] +#![feature(mutex_unpoison)] #![allow(rustc::default_hash_types)] #![allow(rustc::potential_query_instability)] #![deny(rustc::untranslatable_diagnostic)] diff --git a/compiler/rustc_data_structures/src/sharded.rs b/compiler/rustc_data_structures/src/sharded.rs index 7ed70ba1e0fc7..beebf67f5cf8d 100644 --- a/compiler/rustc_data_structures/src/sharded.rs +++ b/compiler/rustc_data_structures/src/sharded.rs @@ -3,22 +3,14 @@ use crate::sync::{CacheAligned, Lock, LockGuard}; use std::borrow::Borrow; use std::collections::hash_map::RawEntryMut; use std::hash::{Hash, Hasher}; -use std::mem; -#[cfg(parallel_compiler)] -// 32 shards is sufficient to reduce contention on an 8-core Ryzen 7 1700, -// but this should be tested on higher core count CPUs. How the `Sharded` type gets used -// may also affect the ideal number of shards. -const SHARD_BITS: usize = 5; - -#[cfg(not(parallel_compiler))] const SHARD_BITS: usize = 0; pub const SHARDS: usize = 1 << SHARD_BITS; /// An array of cache-line aligned inner locked structures with convenience methods. pub struct Sharded { - shards: [CacheAligned>; SHARDS], + shard: Lock, } impl Default for Sharded { @@ -28,34 +20,43 @@ impl Default for Sharded { } } -impl Sharded { +impl Sharded { #[inline] pub fn new(mut value: impl FnMut() -> T) -> Self { - Sharded { shards: [(); SHARDS].map(|()| CacheAligned(Lock::new(value()))) } + Sharded { shard: Lock::new(value()) } } /// The shard is selected by hashing `val` with `FxHasher`. #[inline] - pub fn get_shard_by_value(&self, val: &K) -> &Lock { - if SHARDS == 1 { &self.shards[0].0 } else { self.get_shard_by_hash(make_hash(val)) } + pub fn with_get_shard_by_value R, R>( + &self, + _val: &K, + f: F, + ) -> R { + self.shard.with_lock(f) } #[inline] - pub fn get_shard_by_hash(&self, hash: u64) -> &Lock { - &self.shards[get_shard_index_by_hash(hash)].0 + pub fn with_get_shard_by_hash R, R>(&self, _hash: u64, f: F) -> R { + self.shard.with_lock(f) } #[inline] - pub fn get_shard_by_index(&self, i: usize) -> &Lock { - &self.shards[i].0 + pub fn get_shard_by_value(&self, _val: &K) -> &Lock { + &self.shard + } + + #[inline] + pub fn get_shard_by_hash(&self, _hash: u64) -> &Lock { + &self.shard } pub fn lock_shards(&self) -> Vec> { - (0..SHARDS).map(|i| self.shards[i].0.lock()).collect() + vec![self.shard.lock()] } pub fn try_lock_shards(&self) -> Option>> { - (0..SHARDS).map(|i| self.shards[i].0.try_lock()).collect() + Some(vec![self.shard.try_lock()?]) } } @@ -75,17 +76,18 @@ impl ShardedHashMap { Q: Hash + Eq, { let hash = make_hash(value); - let mut shard = self.get_shard_by_hash(hash).lock(); - let entry = shard.raw_entry_mut().from_key_hashed_nocheck(hash, value); - - match entry { - RawEntryMut::Occupied(e) => *e.key(), - RawEntryMut::Vacant(e) => { - let v = make(); - e.insert_hashed_nocheck(hash, v, ()); - v + self.with_get_shard_by_hash(hash, |shard| { + let entry = shard.raw_entry_mut().from_key_hashed_nocheck(hash, value); + + match entry { + RawEntryMut::Occupied(e) => *e.key(), + RawEntryMut::Vacant(e) => { + let v = make(); + e.insert_hashed_nocheck(hash, v, ()); + v + } } - } + }) } #[inline] @@ -95,17 +97,18 @@ impl ShardedHashMap { Q: Hash + Eq, { let hash = make_hash(&value); - let mut shard = self.get_shard_by_hash(hash).lock(); - let entry = shard.raw_entry_mut().from_key_hashed_nocheck(hash, &value); - - match entry { - RawEntryMut::Occupied(e) => *e.key(), - RawEntryMut::Vacant(e) => { - let v = make(value); - e.insert_hashed_nocheck(hash, v, ()); - v + self.with_get_shard_by_hash(hash, |shard| { + let entry = shard.raw_entry_mut().from_key_hashed_nocheck(hash, &value); + + match entry { + RawEntryMut::Occupied(e) => *e.key(), + RawEntryMut::Vacant(e) => { + let v = make(value); + e.insert_hashed_nocheck(hash, v, ()); + v + } } - } + }) } } @@ -117,9 +120,11 @@ pub trait IntoPointer { impl ShardedHashMap { pub fn contains_pointer_to(&self, value: &T) -> bool { let hash = make_hash(&value); - let shard = self.get_shard_by_hash(hash).lock(); - let value = value.into_pointer(); - shard.raw_entry().from_hash(hash, |entry| entry.into_pointer() == value).is_some() + + self.with_get_shard_by_hash(hash, |shard| { + let value = value.into_pointer(); + shard.raw_entry().from_hash(hash, |entry| entry.into_pointer() == value).is_some() + }) } } @@ -130,6 +135,7 @@ pub fn make_hash(val: &K) -> u64 { state.finish() } +/* /// Get a shard with a pre-computed hash value. If `get_shard_by_value` is /// ever used in combination with `get_shard_by_hash` on a single `Sharded` /// instance, then `hash` must be computed with `FxHasher`. Otherwise, @@ -137,10 +143,11 @@ pub fn make_hash(val: &K) -> u64 { /// consistently for each `Sharded` instance. #[inline] #[allow(clippy::modulo_one)] -pub fn get_shard_index_by_hash(hash: u64) -> usize { +fn get_shard_index_by_hash(hash: u64) -> usize { let hash_len = mem::size_of::(); // Ignore the top 7 bits as hashbrown uses these and get the next SHARD_BITS highest bits. // hashbrown also uses the lowest bits, so we can't use those let bits = (hash >> (hash_len * 8 - 7 - SHARD_BITS)) as usize; bits % SHARDS } +*/ From 3b0715f16581c9dac3c79f3dae01f2b53821a147 Mon Sep 17 00:00:00 2001 From: SparrowLii Date: Sun, 23 Apr 2023 10:41:56 +0800 Subject: [PATCH 03/11] set parallel_compiler to default --- compiler/rustc_codegen_ssa/src/base.rs | 2 +- compiler/rustc_driver_impl/src/lib.rs | 2 +- compiler/rustc_interface/src/interface.rs | 4 +- compiler/rustc_interface/src/util.rs | 7 +- compiler/rustc_query_system/src/query/job.rs | 1 - .../rustc_query_system/src/query/plumbing.rs | 88 +++++++------------ src/bootstrap/config.rs | 2 +- src/librustdoc/lib.rs | 4 +- 8 files changed, 43 insertions(+), 67 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index d9b0a15259465..a6fad07f2a2ae 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -686,7 +686,7 @@ pub fn codegen_crate( // This likely is a temporary measure. Once we don't have to support the // non-parallel compiler anymore, we can compile CGUs end-to-end in // parallel and get rid of the complicated scheduling logic. - let mut pre_compiled_cgus = if tcx.sess.threads() > 1 { + let mut pre_compiled_cgus = if rustc_data_structures::sync::active() { tcx.sess.time("compile_first_CGU_batch", || { // Try to find one CGU to compile per thread. let cgus: Vec<_> = cgu_reuse diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs index 80a9dfd251a79..9846d721616ac 100644 --- a/compiler/rustc_driver_impl/src/lib.rs +++ b/compiler/rustc_driver_impl/src/lib.rs @@ -257,7 +257,7 @@ fn run_compiler( let sopts = config::build_session_options(&matches); // Set parallel mode before thread pool creation, which will create `Lock`s. - interface::set_thread_safe_mode(&sopts.unstable_opts); + interface::set_parallel_mode(&sopts.unstable_opts); if let Some(ref code) = matches.opt_str("explain") { handle_explain(diagnostics_registry(), code, sopts.error_format); diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs index c9e857141c92a..6417a5cd722cf 100644 --- a/compiler/rustc_interface/src/interface.rs +++ b/compiler/rustc_interface/src/interface.rs @@ -61,8 +61,8 @@ impl Compiler { } #[allow(rustc::bad_opt_access)] -pub fn set_thread_safe_mode(sopts: &config::UnstableOptions) { - rustc_data_structures::sync::set_dyn_thread_safe_mode(sopts.threads > 1); +pub fn set_parallel_mode(sopts: &config::UnstableOptions) { + rustc_data_structures::sync::set(sopts.threads > 1); } /// Converts strings provided as `--cfg [cfgspec]` into a `crate_cfg`. diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs index 8d37b1053d800..d0336063024c3 100644 --- a/compiler/rustc_interface/src/util.rs +++ b/compiler/rustc_interface/src/util.rs @@ -128,12 +128,15 @@ fn get_stack_size() -> Option { env::var_os("RUST_MIN_STACK").is_none().then_some(STACK_SIZE) } -#[cfg(not(parallel_compiler))] pub(crate) fn run_in_thread_pool_with_globals R + Send, R: Send>( edition: Edition, _threads: usize, f: F, ) -> R { + #[cfg(parallel_compiler)] + if _threads > 1 { + return run_in_threads_pool_with_globals(edition, _threads, f); + } // The "thread pool" is a single spawned thread in the non-parallel // compiler. We run on a spawned thread instead of the main thread (a) to // provide control over the stack size, and (b) to increase similarity with @@ -163,7 +166,7 @@ pub(crate) fn run_in_thread_pool_with_globals R + Send, R: Send>( } #[cfg(parallel_compiler)] -pub(crate) fn run_in_thread_pool_with_globals R + Send, R: Send>( +pub(crate) fn run_in_threads_pool_with_globals R + Send, R: Send>( edition: Edition, threads: usize, f: F, diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs index 5f2ec656d1d1e..da12f6c3f7b9d 100644 --- a/compiler/rustc_query_system/src/query/job.rs +++ b/compiler/rustc_query_system/src/query/job.rs @@ -124,7 +124,6 @@ impl QueryJob { } impl QueryJobId { - #[cfg(not(parallel_compiler))] pub(super) fn find_cycle_in_stack( &self, query_map: QueryMap, diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index dbfe62ae6e943..bdd8fbbad7e9d 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -12,12 +12,14 @@ use crate::query::job::{report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobI use crate::query::SerializedDepNodeIndex; use crate::query::{QueryContext, QueryMap, QuerySideEffects, QueryStackFrame}; use crate::HandleCycleError; +#[cfg(parallel_compiler)] +use rustc_data_structures::cold_path; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sharded::Sharded; use rustc_data_structures::stack::ensure_sufficient_stack; use rustc_data_structures::sync::Lock; -#[cfg(parallel_compiler)] -use rustc_data_structures::{cold_path, sharded::Sharded}; + use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed, FatalError}; use rustc_span::{Span, DUMMY_SP}; use std::cell::Cell; @@ -25,15 +27,13 @@ use std::collections::hash_map::Entry; use std::fmt::Debug; use std::hash::Hash; use std::mem; +use std::ops::DerefMut; use thin_vec::ThinVec; use super::QueryConfig; pub struct QueryState { - #[cfg(parallel_compiler)] active: Sharded>>, - #[cfg(not(parallel_compiler))] - active: Lock>>, } /// Indicates the state of a query for a given key in a query map. @@ -52,15 +52,8 @@ where D: DepKind, { pub fn all_inactive(&self) -> bool { - #[cfg(parallel_compiler)] - { - let shards = self.active.lock_shards(); - shards.iter().all(|shard| shard.is_empty()) - } - #[cfg(not(parallel_compiler))] - { - self.active.lock().is_empty() - } + let shards = self.active.lock_shards(); + shards.iter().all(|shard| shard.is_empty()) } pub fn try_collect_active_jobs( @@ -69,27 +62,11 @@ where make_query: fn(Qcx, K) -> QueryStackFrame, jobs: &mut QueryMap, ) -> Option<()> { - #[cfg(parallel_compiler)] - { - // We use try_lock_shards here since we are called from the - // deadlock handler, and this shouldn't be locked. - let shards = self.active.try_lock_shards()?; - for shard in shards.iter() { - for (k, v) in shard.iter() { - if let QueryResult::Started(ref job) = *v { - let query = make_query(qcx, *k); - jobs.insert(job.id, QueryJobInfo { query, job: job.clone() }); - } - } - } - } - #[cfg(not(parallel_compiler))] - { - // We use try_lock here since we are called from the - // deadlock handler, and this shouldn't be locked. - // (FIXME: Is this relevant for non-parallel compilers? It doesn't - // really hurt much.) - for (k, v) in self.active.try_lock()?.iter() { + // We use try_lock_shards here since we are called from the + // deadlock handler, and this shouldn't be locked. + let shards = self.active.try_lock_shards()?; + for shard in shards.iter() { + for (k, v) in shard.iter() { if let QueryResult::Started(ref job) = *v { let query = make_query(qcx, *k); jobs.insert(job.id, QueryJobInfo { query, job: job.clone() }); @@ -183,14 +160,10 @@ where cache.complete(key, result, dep_node_index); let job = { - #[cfg(parallel_compiler)] - let mut lock = state.active.get_shard_by_value(&key).lock(); - #[cfg(not(parallel_compiler))] - let mut lock = state.active.lock(); - match lock.remove(&key).unwrap() { + state.active.with_get_shard_by_value(&key, |lock| match lock.remove(&key).unwrap() { QueryResult::Started(job) => job, QueryResult::Poisoned => panic!(), - } + }) }; job.signal_complete(); @@ -208,16 +181,14 @@ where // Poison the query so jobs waiting on it panic. let state = self.state; let job = { - #[cfg(parallel_compiler)] - let mut shard = state.active.get_shard_by_value(&self.key).lock(); - #[cfg(not(parallel_compiler))] - let mut shard = state.active.lock(); - let job = match shard.remove(&self.key).unwrap() { - QueryResult::Started(job) => job, - QueryResult::Poisoned => panic!(), - }; - shard.insert(self.key, QueryResult::Poisoned); - job + state.active.with_get_shard_by_value(&self.key, |shard| { + let job = match shard.remove(&self.key).unwrap() { + QueryResult::Started(job) => job, + QueryResult::Poisoned => panic!(), + }; + shard.insert(self.key, QueryResult::Poisoned); + job + }) }; // Also signal the completion of the job, so waiters // will continue execution. @@ -254,7 +225,6 @@ where #[cold] #[inline(never)] -#[cfg(not(parallel_compiler))] fn cycle_error( query: Q, qcx: Qcx, @@ -324,10 +294,8 @@ where Qcx: QueryContext, { let state = query.query_state(qcx); - #[cfg(parallel_compiler)] let mut state_lock = state.active.get_shard_by_value(&key).lock(); - #[cfg(not(parallel_compiler))] - let mut state_lock = state.active.lock(); + let lock = state_lock.deref_mut(); // For the parallel compiler we need to check both the query cache and query state structures // while holding the state lock to ensure that 1) the query has not yet completed and 2) the @@ -344,7 +312,7 @@ where let current_job_id = qcx.current_query_job(); - match state_lock.entry(key) { + match lock.entry(key) { Entry::Vacant(entry) => { // Nothing has computed or is computing the query, so we start a new job and insert it in the // state map. @@ -370,6 +338,14 @@ where } #[cfg(parallel_compiler)] QueryResult::Started(job) => { + if std::intrinsics::likely(!rustc_data_structures::sync::active()) { + let id = job.id; + drop(state_lock); + + // If we are single-threaded we know that we have cycle error, + // so we just return the error. + return cycle_error(query, qcx, id, span); + } // Get the latch out let latch = job.latch(); drop(state_lock); diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 710c8b52194b4..e51fe601149db 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -1147,7 +1147,7 @@ impl Config { set(&mut config.use_lld, rust.use_lld); set(&mut config.lld_enabled, rust.lld); set(&mut config.llvm_tools_enabled, rust.llvm_tools); - config.rustc_parallel = rust.parallel_compiler.unwrap_or(false); + config.rustc_parallel = rust.parallel_compiler.unwrap_or(true); config.rustc_default_linker = rust.default_linker; config.musl_root = rust.musl_root.map(PathBuf::from); config.save_toolstates = rust.save_toolstates.map(PathBuf::from); diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs index 5460bce21a512..369de24bffeb3 100644 --- a/src/librustdoc/lib.rs +++ b/src/librustdoc/lib.rs @@ -212,8 +212,6 @@ fn init_logging() { .with_verbose_exit(true) .with_verbose_entry(true) .with_indent_amount(2); - #[cfg(all(parallel_compiler, debug_assertions))] - let layer = layer.with_thread_ids(true).with_thread_names(true); use tracing_subscriber::layer::SubscriberExt; let subscriber = tracing_subscriber::Registry::default().with(filter).with(layer); @@ -740,7 +738,7 @@ fn main_args(at_args: &[String]) -> MainResult { }; // Set parallel mode before error handler creation, which will create `Lock`s. - interface::set_thread_safe_mode(&options.unstable_opts); + interface::set_parallel_mode(&options.unstable_opts); let diag = core::new_handler( options.error_format, From 3ad4a0eb3b5885aba750cd74f552345139c7310d Mon Sep 17 00:00:00 2001 From: SparrowLii Date: Sun, 23 Apr 2023 10:42:53 +0800 Subject: [PATCH 04/11] use `with_lock` tp avoid drop --- compiler/rustc_query_system/src/cache.rs | 6 +- .../rustc_query_system/src/dep_graph/graph.rs | 164 +++++++++--------- .../src/dep_graph/serialized.rs | 6 +- .../src/ich/impls_syntax.rs | 2 +- .../rustc_query_system/src/query/caches.rs | 78 ++------- compiler/rustc_session/src/parse.rs | 22 ++- 6 files changed, 121 insertions(+), 157 deletions(-) diff --git a/compiler/rustc_query_system/src/cache.rs b/compiler/rustc_query_system/src/cache.rs index 6e862db0b2547..e06579eb0ba63 100644 --- a/compiler/rustc_query_system/src/cache.rs +++ b/compiler/rustc_query_system/src/cache.rs @@ -26,17 +26,17 @@ impl Default for Cache { impl Cache { /// Actually frees the underlying memory in contrast to what stdlib containers do on `clear` pub fn clear(&self) { - *self.hashmap.borrow_mut() = Default::default(); + self.hashmap.with_lock(|map| *map = Default::default()); } } impl Cache { pub fn get(&self, key: &Key, tcx: Tcx) -> Option { - Some(self.hashmap.borrow().get(key)?.get(tcx)) + self.hashmap.with_borrow(|map| map.get(key).map(|node| node.get(tcx))) } pub fn insert(&self, key: Key, dep_node: DepNodeIndex, value: Value) { - self.hashmap.borrow_mut().insert(key, WithDepNode::new(dep_node, value)); + self.hashmap.with_lock(|map| map.insert(key, WithDepNode::new(dep_node, value))); } } diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index 8de4d06fe782b..42fd40c30a4e3 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -2,7 +2,7 @@ use parking_lot::Mutex; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::profiling::{EventId, QueryInvocationId, SelfProfilerRef}; -use rustc_data_structures::sharded::{self, Sharded}; +use rustc_data_structures::sharded::Sharded; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::steal::Steal; use rustc_data_structures::sync::{AtomicU32, AtomicU64, Lock, Lrc, Ordering}; @@ -466,8 +466,42 @@ impl DepGraph { pub fn read_index(&self, dep_node_index: DepNodeIndex) { if let Some(ref data) = self.data { K::read_deps(|task_deps| { - let mut task_deps = match task_deps { - TaskDepsRef::Allow(deps) => deps.lock(), + match task_deps { + TaskDepsRef::Allow(deps) => deps.with_lock(|task_deps| { + // As long as we only have a low number of reads we can avoid doing a hash + // insert and potentially allocating/reallocating the hashmap + let new_read = if task_deps.reads.len() < TASK_DEPS_READS_CAP { + task_deps.reads.iter().all(|other| *other != dep_node_index) + } else { + task_deps.read_set.insert(dep_node_index) + }; + if new_read { + task_deps.reads.push(dep_node_index); + if task_deps.reads.len() == TASK_DEPS_READS_CAP { + // Fill `read_set` with what we have so far so we can use the hashset + // next time + task_deps.read_set.extend(task_deps.reads.iter().copied()); + } + + #[cfg(debug_assertions)] + { + if let Some(target) = task_deps.node { + if let Some(ref forbidden_edge) = data.current.forbidden_edge { + let src = + forbidden_edge.index_to_node.lock()[&dep_node_index]; + if forbidden_edge.test(&src, &target) { + panic!( + "forbidden edge {:?} -> {:?} created", + src, target + ) + } + } + } + } + } else if cfg!(debug_assertions) { + data.current.total_duplicate_read_count.fetch_add(1, Relaxed); + } + }), TaskDepsRef::EvalAlways => { // We don't need to record dependencies of eval_always // queries. They are re-evaluated unconditionally anyway. @@ -478,41 +512,6 @@ impl DepGraph { panic!("Illegal read of: {dep_node_index:?}") } }; - let task_deps = &mut *task_deps; - - if cfg!(debug_assertions) { - data.current.total_read_count.fetch_add(1, Relaxed); - } - - // As long as we only have a low number of reads we can avoid doing a hash - // insert and potentially allocating/reallocating the hashmap - let new_read = if task_deps.reads.len() < TASK_DEPS_READS_CAP { - task_deps.reads.iter().all(|other| *other != dep_node_index) - } else { - task_deps.read_set.insert(dep_node_index) - }; - if new_read { - task_deps.reads.push(dep_node_index); - if task_deps.reads.len() == TASK_DEPS_READS_CAP { - // Fill `read_set` with what we have so far so we can use the hashset - // next time - task_deps.read_set.extend(task_deps.reads.iter().copied()); - } - - #[cfg(debug_assertions)] - { - if let Some(target) = task_deps.node { - if let Some(ref forbidden_edge) = data.current.forbidden_edge { - let src = forbidden_edge.index_to_node.lock()[&dep_node_index]; - if forbidden_edge.test(&src, &target) { - panic!("forbidden edge {:?} -> {:?} created", src, target) - } - } - } - } - } else if cfg!(debug_assertions) { - data.current.total_duplicate_read_count.fetch_add(1, Relaxed); - } }) } } @@ -574,7 +573,9 @@ impl DepGraph { let mut edges = SmallVec::new(); K::read_deps(|task_deps| match task_deps { - TaskDepsRef::Allow(deps) => edges.extend(deps.lock().reads.iter().copied()), + TaskDepsRef::Allow(deps) => { + deps.with_borrow(|deps| edges.extend(deps.reads.iter().copied())) + } TaskDepsRef::EvalAlways => { edges.push(DepNodeIndex::FOREVER_RED_NODE); } @@ -627,14 +628,11 @@ impl DepGraphData { #[inline] pub fn dep_node_index_of_opt(&self, dep_node: &DepNode) -> Option { if let Some(prev_index) = self.previous.node_to_index_opt(dep_node) { - self.current.prev_index_to_index.lock()[prev_index] + self.current.prev_index_to_index.with_borrow(|nodes| nodes[prev_index]) } else { self.current .new_node_to_index - .get_shard_by_value(dep_node) - .lock() - .get(dep_node) - .copied() + .with_get_shard_by_value(dep_node, |node| node.get(dep_node).copied()) } } @@ -670,7 +668,7 @@ impl DepGraphData { } pub fn mark_debug_loaded_from_disk(&self, dep_node: DepNode) { - self.debug_loaded_from_disk.lock().insert(dep_node); + self.debug_loaded_from_disk.with_lock(|node| node.insert(dep_node)); } } @@ -693,7 +691,11 @@ impl DepGraph { } pub fn debug_was_loaded_from_disk(&self, dep_node: DepNode) -> bool { - self.data.as_ref().unwrap().debug_loaded_from_disk.lock().contains(&dep_node) + self.data + .as_ref() + .unwrap() + .debug_loaded_from_disk + .with_borrow(|node| node.contains(&dep_node)) } #[inline(always)] @@ -703,15 +705,15 @@ impl DepGraph { { let dep_node_debug = &self.data.as_ref().unwrap().dep_node_debug; - if dep_node_debug.borrow().contains_key(&dep_node) { + if dep_node_debug.with_borrow(|node| node.contains_key(&dep_node)) { return; } let debug_str = self.with_ignore(debug_str_gen); - dep_node_debug.borrow_mut().insert(dep_node, debug_str); + dep_node_debug.with_lock(|node| node.insert(dep_node, debug_str)); } pub fn dep_node_debug_str(&self, dep_node: DepNode) -> Option { - self.data.as_ref()?.dep_node_debug.borrow().get(&dep_node).cloned() + self.data.as_ref()?.dep_node_debug.with_borrow(|node| node.get(&dep_node).cloned()) } fn node_color(&self, dep_node: &DepNode) -> Option { @@ -1163,10 +1165,7 @@ impl CurrentDepGraph { record_stats, )), new_node_to_index: Sharded::new(|| { - FxHashMap::with_capacity_and_hasher( - new_node_count_estimate / sharded::SHARDS, - Default::default(), - ) + FxHashMap::with_capacity_and_hasher(new_node_count_estimate, Default::default()) }), prev_index_to_index: Lock::new(IndexVec::from_elem_n(None, prev_graph_node_count)), anon_id_seed, @@ -1199,16 +1198,16 @@ impl CurrentDepGraph { edges: EdgesVec, current_fingerprint: Fingerprint, ) -> DepNodeIndex { - let dep_node_index = match self.new_node_to_index.get_shard_by_value(&key).lock().entry(key) - { - Entry::Occupied(entry) => *entry.get(), - Entry::Vacant(entry) => { - let dep_node_index = - self.encoder.borrow().send(profiler, key, current_fingerprint, edges); - entry.insert(dep_node_index); - dep_node_index - } - }; + let dep_node_index = + self.new_node_to_index.with_get_shard_by_value(&key, |node| match node.entry(key) { + Entry::Occupied(entry) => *entry.get(), + Entry::Vacant(entry) => { + let dep_node_index = + self.encoder.borrow().send(profiler, key, current_fingerprint, edges); + entry.insert(dep_node_index); + dep_node_index + } + }); #[cfg(debug_assertions)] self.record_edge(dep_node_index, key, current_fingerprint); @@ -1298,25 +1297,26 @@ impl CurrentDepGraph { ) -> DepNodeIndex { self.debug_assert_not_in_new_nodes(prev_graph, prev_index); - let mut prev_index_to_index = self.prev_index_to_index.lock(); - - match prev_index_to_index[prev_index] { - Some(dep_node_index) => dep_node_index, - None => { - let key = prev_graph.index_to_node(prev_index); - let edges = prev_graph - .edge_targets_from(prev_index) - .iter() - .map(|i| prev_index_to_index[*i].unwrap()) - .collect(); - let fingerprint = prev_graph.fingerprint_by_index(prev_index); - let dep_node_index = self.encoder.borrow().send(profiler, key, fingerprint, edges); - prev_index_to_index[prev_index] = Some(dep_node_index); - #[cfg(debug_assertions)] - self.record_edge(dep_node_index, key, fingerprint); - dep_node_index + self.prev_index_to_index.with_lock(|prev_index_to_index| { + match prev_index_to_index[prev_index] { + Some(dep_node_index) => dep_node_index, + None => { + let key = prev_graph.index_to_node(prev_index); + let edges = prev_graph + .edge_targets_from(prev_index) + .iter() + .map(|i| prev_index_to_index[*i].unwrap()) + .collect(); + let fingerprint = prev_graph.fingerprint_by_index(prev_index); + let dep_node_index = + self.encoder.borrow().send(profiler, key, fingerprint, edges); + prev_index_to_index[prev_index] = Some(dep_node_index); + #[cfg(debug_assertions)] + self.record_edge(dep_node_index, key, fingerprint); + dep_node_index + } } - } + }) } #[inline] @@ -1327,7 +1327,7 @@ impl CurrentDepGraph { ) { let node = &prev_graph.index_to_node(prev_index); debug_assert!( - !self.new_node_to_index.get_shard_by_value(node).lock().contains_key(node), + !self.new_node_to_index.with_get_shard_by_value(node, |lock| lock.contains_key(node)), "node from previous graph present in new node collection" ); } diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs index edddfda624242..65b64562e42b7 100644 --- a/compiler/rustc_query_system/src/dep_graph/serialized.rs +++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs @@ -186,7 +186,7 @@ impl EncoderState { if let Some(record_graph) = &mut record_graph.try_lock() { record_graph.push(index, node.node, &node.edges); } - } + }; if let Some(stats) = &mut self.stats { let kind = node.node.kind; @@ -242,7 +242,7 @@ impl> GraphEncoder { pub(crate) fn with_query(&self, f: impl Fn(&DepGraphQuery)) { if let Some(record_graph) = &self.record_graph { - f(&record_graph.lock()) + record_graph.with_borrow(f) } } @@ -307,7 +307,7 @@ impl> GraphEncoder { ) -> DepNodeIndex { let _prof_timer = profiler.generic_activity("incr_comp_encode_dep_graph"); let node = NodeInfo { node, fingerprint, edges }; - self.status.lock().encode_node(&node, &self.record_graph) + self.status.with_lock(|status| status.encode_node(&node, &self.record_graph)) } pub fn finish(self, profiler: &SelfProfilerRef) -> FileEncodeResult { diff --git a/compiler/rustc_query_system/src/ich/impls_syntax.rs b/compiler/rustc_query_system/src/ich/impls_syntax.rs index 8865ecf3e054a..feedc46e1b31b 100644 --- a/compiler/rustc_query_system/src/ich/impls_syntax.rs +++ b/compiler/rustc_query_system/src/ich/impls_syntax.rs @@ -80,7 +80,7 @@ impl<'a> HashStable> for SourceFile { src_hash.hash_stable(hcx, hasher); // We are always in `Lines` form by the time we reach here. - assert!(self.lines.borrow().is_lines()); + assert!(self.lines.with_borrow(|lines| lines.is_lines())); self.lines(|lines| { // We only hash the relative position within this source_file lines.len().hash_stable(hcx, hasher); diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs index 9a09f516ec920..d17711d13630d 100644 --- a/compiler/rustc_query_system/src/query/caches.rs +++ b/compiler/rustc_query_system/src/query/caches.rs @@ -2,7 +2,6 @@ use crate::dep_graph::DepNodeIndex; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sharded; -#[cfg(parallel_compiler)] use rustc_data_structures::sharded::Sharded; use rustc_data_structures::sync::Lock; use rustc_index::{Idx, IndexVec}; @@ -37,10 +36,7 @@ impl<'tcx, K: Eq + Hash, V: 'tcx> CacheSelector<'tcx, V> for DefaultCacheSelecto } pub struct DefaultCache { - #[cfg(parallel_compiler)] cache: Sharded>, - #[cfg(not(parallel_compiler))] - cache: Lock>, } impl Default for DefaultCache { @@ -60,40 +56,23 @@ where #[inline(always)] fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> { let key_hash = sharded::make_hash(key); - #[cfg(parallel_compiler)] - let lock = self.cache.get_shard_by_hash(key_hash).lock(); - #[cfg(not(parallel_compiler))] - let lock = self.cache.lock(); - let result = lock.raw_entry().from_key_hashed_nocheck(key_hash, key); - - if let Some((_, value)) = result { Some(*value) } else { None } + self.cache.with_get_shard_by_hash(key_hash, |lock| { + let result = lock.raw_entry().from_key_hashed_nocheck(key_hash, key); + if let Some((_, value)) = result { Some(*value) } else { None } + }) } #[inline] fn complete(&self, key: K, value: V, index: DepNodeIndex) { - #[cfg(parallel_compiler)] - let mut lock = self.cache.get_shard_by_value(&key).lock(); - #[cfg(not(parallel_compiler))] - let mut lock = self.cache.lock(); // We may be overwriting another value. This is all right, since the dep-graph // will check that the fingerprint matches. - lock.insert(key, (value, index)); + self.cache.with_get_shard_by_value(&key, |cache| cache.insert(key, (value, index))); } fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) { - #[cfg(parallel_compiler)] - { - let shards = self.cache.lock_shards(); - for shard in shards.iter() { - for (k, v) in shard.iter() { - f(k, &v.0, v.1); - } - } - } - #[cfg(not(parallel_compiler))] - { - let map = self.cache.lock(); - for (k, v) in map.iter() { + let shards = self.cache.lock_shards(); + for shard in shards.iter() { + for (k, v) in shard.iter() { f(k, &v.0, v.1); } } @@ -127,12 +106,12 @@ where #[inline(always)] fn lookup(&self, _key: &()) -> Option<(V, DepNodeIndex)> { - *self.cache.lock() + self.cache.with_lock(|cache| *cache) } #[inline] fn complete(&self, _key: (), value: V, index: DepNodeIndex) { - *self.cache.lock() = Some((value, index)); + self.cache.with_lock(|cache| *cache = Some((value, index))); } fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) { @@ -151,10 +130,7 @@ impl<'tcx, K: Idx, V: 'tcx> CacheSelector<'tcx, V> for VecCacheSelector { } pub struct VecCache { - #[cfg(parallel_compiler)] cache: Sharded>>, - #[cfg(not(parallel_compiler))] - cache: Lock>>, } impl Default for VecCache { @@ -173,38 +149,22 @@ where #[inline(always)] fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> { - #[cfg(parallel_compiler)] - let lock = self.cache.get_shard_by_hash(key.index() as u64).lock(); - #[cfg(not(parallel_compiler))] - let lock = self.cache.lock(); - if let Some(Some(value)) = lock.get(*key) { Some(*value) } else { None } + self.cache.with_get_shard_by_hash(key.index() as u64, |lock| { + if let Some(Some(value)) = lock.get(*key) { Some(*value) } else { None } + }) } #[inline] fn complete(&self, key: K, value: V, index: DepNodeIndex) { - #[cfg(parallel_compiler)] - let mut lock = self.cache.get_shard_by_hash(key.index() as u64).lock(); - #[cfg(not(parallel_compiler))] - let mut lock = self.cache.lock(); - lock.insert(key, (value, index)); + self.cache.with_get_shard_by_hash(key.index() as u64, |lock| { + lock.insert(key, (value, index)); + }) } fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) { - #[cfg(parallel_compiler)] - { - let shards = self.cache.lock_shards(); - for shard in shards.iter() { - for (k, v) in shard.iter_enumerated() { - if let Some(v) = v { - f(&k, &v.0, v.1); - } - } - } - } - #[cfg(not(parallel_compiler))] - { - let map = self.cache.lock(); - for (k, v) in map.iter_enumerated() { + let shards = self.cache.lock_shards(); + for shard in shards.iter() { + for (k, v) in shard.iter_enumerated() { if let Some(v) = v { f(&k, &v.0, v.1); } diff --git a/compiler/rustc_session/src/parse.rs b/compiler/rustc_session/src/parse.rs index 5cc9c62617dd5..5ca5dbb12177c 100644 --- a/compiler/rustc_session/src/parse.rs +++ b/compiler/rustc_session/src/parse.rs @@ -39,7 +39,7 @@ impl GatedSpans { /// Feature gate the given `span` under the given `feature` /// which is same `Symbol` used in `active.rs`. pub fn gate(&self, feature: Symbol, span: Span) { - self.spans.borrow_mut().entry(feature).or_default().push(span); + self.spans.with_lock(|spans| spans.entry(feature).or_default().push(span)); } /// Ungate the last span under the given `feature`. @@ -47,7 +47,8 @@ impl GatedSpans { /// /// Using this is discouraged unless you have a really good reason to. pub fn ungate_last(&self, feature: Symbol, span: Span) { - let removed_span = self.spans.borrow_mut().entry(feature).or_default().pop().unwrap(); + let removed_span = + self.spans.with_lock(|spans| spans.entry(feature).or_default().pop().unwrap()); debug_assert_eq!(span, removed_span); } @@ -55,16 +56,17 @@ impl GatedSpans { /// /// Using this is discouraged unless you have a really good reason to. pub fn is_ungated(&self, feature: Symbol) -> bool { - self.spans.borrow().get(&feature).map_or(true, |spans| spans.is_empty()) + self.spans.with_borrow(|spans| spans.get(&feature).map_or(true, |spans| spans.is_empty())) } /// Prepend the given set of `spans` onto the set in `self`. pub fn merge(&self, mut spans: FxHashMap>) { - let mut inner = self.spans.borrow_mut(); - for (gate, mut gate_spans) in inner.drain() { - spans.entry(gate).or_default().append(&mut gate_spans); - } - *inner = spans; + self.spans.with_lock(|inner| { + for (gate, mut gate_spans) in inner.drain() { + spans.entry(gate).or_default().append(&mut gate_spans); + } + *inner = spans; + }) } } @@ -78,7 +80,9 @@ impl SymbolGallery { /// Insert a symbol and its span into symbol gallery. /// If the symbol has occurred before, ignore the new occurrence. pub fn insert(&self, symbol: Symbol, span: Span) { - self.symbols.lock().entry(symbol).or_insert(span); + self.symbols.with_lock(|symbols| { + symbols.entry(symbol).or_insert(span); + }) } } From eaf5ee628812de567e582fb5b943aec093a9082f Mon Sep 17 00:00:00 2001 From: SparrowLii Date: Sun, 23 Apr 2023 10:43:18 +0800 Subject: [PATCH 05/11] fix test errors --- .../rustc_const_eval/src/interpret/memory.rs | 25 +++++++++++++++++++ compiler/rustc_const_eval/src/lib.rs | 1 + compiler/rustc_log/src/lib.rs | 2 -- compiler/rustc_middle/src/ty/fold.rs | 16 ++++++++++++ src/librustdoc/clean/utils.rs | 6 +---- src/tools/clippy/clippy_utils/src/lib.rs | 1 + src/tools/clippy/clippy_utils/src/visitors.rs | 12 +++++++++ .../missing-rustc-driver-error.stderr | 22 +++++++++++++++- 8 files changed, 77 insertions(+), 8 deletions(-) diff --git a/compiler/rustc_const_eval/src/interpret/memory.rs b/compiler/rustc_const_eval/src/interpret/memory.rs index d5b6a581a79f6..ef053fef5cdc2 100644 --- a/compiler/rustc_const_eval/src/interpret/memory.rs +++ b/compiler/rustc_const_eval/src/interpret/memory.rs @@ -121,6 +121,21 @@ pub struct AllocRef<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes = Box<[ tcx: TyCtxt<'tcx>, alloc_id: AllocId, } + +// FIXME: More information in . +unsafe impl<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes> Send + for AllocRef<'a, 'tcx, Prov, Extra, Bytes> +where + Allocation: Sync, +{ +} +unsafe impl<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes> Sync + for AllocRef<'a, 'tcx, Prov, Extra, Bytes> +where + Allocation: Sync, +{ +} + /// A reference to some allocation that was already bounds-checked for the given region /// and had the on-access machine hooks run. pub struct AllocRefMut<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes = Box<[u8]>> { @@ -130,6 +145,16 @@ pub struct AllocRefMut<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes = Bo alloc_id: AllocId, } +// FIXME: More information in . +impl<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes> !Send + for AllocRefMut<'a, 'tcx, Prov, Extra, Bytes> +{ +} +impl<'a, 'tcx, Prov: Provenance, Extra, Bytes: AllocBytes> !Sync + for AllocRefMut<'a, 'tcx, Prov, Extra, Bytes> +{ +} + impl<'mir, 'tcx, M: Machine<'mir, 'tcx>> Memory<'mir, 'tcx, M> { pub fn new() -> Self { Memory { diff --git a/compiler/rustc_const_eval/src/lib.rs b/compiler/rustc_const_eval/src/lib.rs index c36282d5ed442..fbe7b51c5a5e1 100644 --- a/compiler/rustc_const_eval/src/lib.rs +++ b/compiler/rustc_const_eval/src/lib.rs @@ -20,6 +20,7 @@ Rust MIR: a lowered representation of Rust. #![feature(try_blocks)] #![feature(yeet_expr)] #![feature(if_let_guard)] +#![feature(negative_impls)] #![recursion_limit = "256"] #[macro_use] diff --git a/compiler/rustc_log/src/lib.rs b/compiler/rustc_log/src/lib.rs index 3cbb2c21e289e..e3d8a31c09080 100644 --- a/compiler/rustc_log/src/lib.rs +++ b/compiler/rustc_log/src/lib.rs @@ -82,8 +82,6 @@ pub fn init_env_logger(env: &str) -> Result<(), Error> { .with_verbose_exit(verbose_entry_exit) .with_verbose_entry(verbose_entry_exit) .with_indent_amount(2); - #[cfg(all(parallel_compiler, debug_assertions))] - let layer = layer.with_thread_ids(true).with_thread_names(true); let subscriber = tracing_subscriber::Registry::default().with(filter).with(layer); match env::var(format!("{env}_BACKTRACE")) { diff --git a/compiler/rustc_middle/src/ty/fold.rs b/compiler/rustc_middle/src/ty/fold.rs index 25890eb15cde4..e597d2bfcc8b7 100644 --- a/compiler/rustc_middle/src/ty/fold.rs +++ b/compiler/rustc_middle/src/ty/fold.rs @@ -21,6 +21,22 @@ where pub ct_op: H, } +// FIXME: More information in . +impl<'tcx, F, G, H> !Send for BottomUpFolder<'tcx, F, G, H> +where + F: FnMut(Ty<'tcx>) -> Ty<'tcx>, + G: FnMut(ty::Region<'tcx>) -> ty::Region<'tcx>, + H: FnMut(ty::Const<'tcx>) -> ty::Const<'tcx>, +{ +} +impl<'tcx, F, G, H> !Sync for BottomUpFolder<'tcx, F, G, H> +where + F: FnMut(Ty<'tcx>) -> Ty<'tcx>, + G: FnMut(ty::Region<'tcx>) -> ty::Region<'tcx>, + H: FnMut(ty::Const<'tcx>) -> ty::Const<'tcx>, +{ +} + impl<'tcx, F, G, H> TypeFolder> for BottomUpFolder<'tcx, F, G, H> where F: FnMut(Ty<'tcx>) -> Ty<'tcx>, diff --git a/src/librustdoc/clean/utils.rs b/src/librustdoc/clean/utils.rs index 17aa6b38e389c..d28b484cf2125 100644 --- a/src/librustdoc/clean/utils.rs +++ b/src/librustdoc/clean/utils.rs @@ -470,11 +470,7 @@ pub(crate) fn get_auto_trait_and_blanket_impls( cx: &mut DocContext<'_>, item_def_id: DefId, ) -> impl Iterator { - // FIXME: To be removed once `parallel_compiler` bugs are fixed! - // More information in . - if cfg!(parallel_compiler) { - return vec![].into_iter().chain(vec![].into_iter()); - } + // FIXME: More information in . let auto_impls = cx .sess() diff --git a/src/tools/clippy/clippy_utils/src/lib.rs b/src/tools/clippy/clippy_utils/src/lib.rs index 964104fc31d0e..f8e559deee8b7 100644 --- a/src/tools/clippy/clippy_utils/src/lib.rs +++ b/src/tools/clippy/clippy_utils/src/lib.rs @@ -2,6 +2,7 @@ #![feature(box_patterns)] #![feature(let_chains)] #![feature(lint_reasons)] +#![feature(negative_impls)] #![feature(never_type)] #![feature(rustc_private)] #![recursion_limit = "512"] diff --git a/src/tools/clippy/clippy_utils/src/visitors.rs b/src/tools/clippy/clippy_utils/src/visitors.rs index 5dcd71cef127e..cb97f95995cf4 100644 --- a/src/tools/clippy/clippy_utils/src/visitors.rs +++ b/src/tools/clippy/clippy_utils/src/visitors.rs @@ -113,6 +113,10 @@ pub fn for_each_expr_with_closures<'tcx, B, C: Continue>( f: F, res: Option, } + // FIXME: more details in #106930 + impl<'tcx, B, F> !Send for V<'tcx, B, F> {} + impl<'tcx, B, F> !Sync for V<'tcx, B, F> {} + impl<'tcx, B, C: Continue, F: FnMut(&'tcx Expr<'tcx>) -> ControlFlow> Visitor<'tcx> for V<'tcx, B, F> { type NestedFilter = nested_filter::OnlyBodies; fn nested_visit_map(&mut self) -> Self::Map { @@ -513,6 +517,10 @@ pub fn for_each_local_use_after_expr<'tcx, B>( res: ControlFlow, f: F, } + // FIXME: more details in #106930 + impl<'cx, 'tcx, B, F> !Send for V<'cx, 'tcx, B, F> {} + impl<'cx, 'tcx, B, F> !Sync for V<'cx, 'tcx, B, F> {} + impl<'cx, 'tcx, F: FnMut(&'tcx Expr<'tcx>) -> ControlFlow, B> Visitor<'tcx> for V<'cx, 'tcx, F, B> { type NestedFilter = nested_filter::OnlyBodies; fn nested_visit_map(&mut self) -> Self::Map { @@ -694,6 +702,10 @@ pub fn for_each_local_assignment<'tcx, B>( res: ControlFlow, f: F, } + // FIXME: more details in #106930 + impl<'cx, 'tcx, F, B> !Send for V<'cx, 'tcx, F, B> {} + impl<'cx, 'tcx, F, B> !Sync for V<'cx, 'tcx, F, B> {} + impl<'cx, 'tcx, F: FnMut(&'tcx Expr<'tcx>) -> ControlFlow, B> Visitor<'tcx> for V<'cx, 'tcx, F, B> { type NestedFilter = nested_filter::OnlyBodies; fn nested_visit_map(&mut self) -> Self::Map { diff --git a/tests/ui-fulldeps/missing-rustc-driver-error.stderr b/tests/ui-fulldeps/missing-rustc-driver-error.stderr index ad03ba0103c52..2553eaf208d81 100644 --- a/tests/ui-fulldeps/missing-rustc-driver-error.stderr +++ b/tests/ui-fulldeps/missing-rustc-driver-error.stderr @@ -20,5 +20,25 @@ error: crate `cfg_if` required to be available in rlib format, but was not found error: crate `libc` required to be available in rlib format, but was not found in this form -error: aborting due to 10 previous errors +error: crate `rayon` required to be available in rlib format, but was not found in this form + +error: crate `either` required to be available in rlib format, but was not found in this form + +error: crate `rayon_core` required to be available in rlib format, but was not found in this form + +error: crate `crossbeam_channel` required to be available in rlib format, but was not found in this form + +error: crate `crossbeam_utils` required to be available in rlib format, but was not found in this form + +error: crate `crossbeam_deque` required to be available in rlib format, but was not found in this form + +error: crate `crossbeam_epoch` required to be available in rlib format, but was not found in this form + +error: crate `scopeguard` required to be available in rlib format, but was not found in this form + +error: crate `memoffset` required to be available in rlib format, but was not found in this form + +error: crate `num_cpus` required to be available in rlib format, but was not found in this form + +error: aborting due to 20 previous errors From 424c216a0de240b3a581133f64b2409b10c709f4 Mon Sep 17 00:00:00 2001 From: SparrowLii Date: Sun, 23 Apr 2023 14:38:56 +0800 Subject: [PATCH 06/11] set parallel_compiler to false --- src/bootstrap/config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index e51fe601149db..710c8b52194b4 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -1147,7 +1147,7 @@ impl Config { set(&mut config.use_lld, rust.use_lld); set(&mut config.lld_enabled, rust.lld); set(&mut config.llvm_tools_enabled, rust.llvm_tools); - config.rustc_parallel = rust.parallel_compiler.unwrap_or(true); + config.rustc_parallel = rust.parallel_compiler.unwrap_or(false); config.rustc_default_linker = rust.default_linker; config.musl_root = rust.musl_root.map(PathBuf::from); config.save_toolstates = rust.save_toolstates.map(PathBuf::from); From f08cea40f940106ffe538cf973db36e841a99547 Mon Sep 17 00:00:00 2001 From: SparrowLii Date: Sat, 6 May 2023 11:24:19 +0800 Subject: [PATCH 07/11] specialize the query caches --- compiler/rustc_data_structures/src/sharded.rs | 2 +- compiler/rustc_data_structures/src/sync.rs | 120 ++++++++++-- .../src/sync/worker_local.rs | 55 +----- compiler/rustc_middle/src/query/plumbing.rs | 182 ++++++++++++------ compiler/rustc_query_impl/src/lib.rs | 6 +- compiler/rustc_query_impl/src/plumbing.rs | 96 +++++++-- .../rustc_query_system/src/query/caches.rs | 98 +++++----- .../rustc_query_system/src/query/config.rs | 18 +- .../rustc_query_system/src/query/plumbing.rs | 17 +- src/bootstrap/config.rs | 2 +- 10 files changed, 396 insertions(+), 200 deletions(-) diff --git a/compiler/rustc_data_structures/src/sharded.rs b/compiler/rustc_data_structures/src/sharded.rs index beebf67f5cf8d..287a74190faa8 100644 --- a/compiler/rustc_data_structures/src/sharded.rs +++ b/compiler/rustc_data_structures/src/sharded.rs @@ -1,5 +1,5 @@ use crate::fx::{FxHashMap, FxHasher}; -use crate::sync::{CacheAligned, Lock, LockGuard}; +use crate::sync::{Lock, LockGuard}; use std::borrow::Borrow; use std::collections::hash_map::RawEntryMut; use std::hash::{Hash, Hasher}; diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs index fe48178361020..a34a3231d4f66 100644 --- a/compiler/rustc_data_structures/src/sync.rs +++ b/compiler/rustc_data_structures/src/sync.rs @@ -40,7 +40,7 @@ //! [^2] `MTLockRef` is a typedef. pub use crate::marker::*; -use std::cell::{Cell, UnsafeCell}; +use std::cell::{Cell, RefCell, RefMut, UnsafeCell}; use std::collections::HashMap; use std::fmt::{Debug, Formatter}; use std::hash::{BuildHasher, Hash}; @@ -50,10 +50,10 @@ use std::ops::{Deref, DerefMut}; use std::panic::{catch_unwind, resume_unwind, AssertUnwindSafe}; mod worker_local; -pub use worker_local::{Registry, WorkerLocal}; pub use std::sync::atomic::Ordering; pub use std::sync::atomic::Ordering::SeqCst; +use std::sync::{Mutex, MutexGuard}; pub use vec::{AppendOnlyIndexVec, AppendOnlyVec}; @@ -559,8 +559,8 @@ impl FromDyn { } } -#[derive(Default)] -#[cfg_attr(parallel_compiler, repr(align(64)))] +#[derive(Default, Debug)] +#[repr(align(64))] pub struct CacheAligned(pub T); pub trait HashMapExt { @@ -768,6 +768,103 @@ impl<'a, T> Drop for LockGuard<'a, T> { } } +pub trait SLock: Copy { + type Lock: LockLike; +} + +pub trait LockLike { + type LockGuard<'a>: DerefMut + where + Self: 'a; + + fn new(val: T) -> Self; + + fn into_inner(self) -> T; + + fn get_mut(&mut self) -> &mut T; + + fn try_lock(&self) -> Option>; + + fn lock(&self) -> Self::LockGuard<'_>; +} + +#[derive(Copy, Clone, Default)] +pub struct SRefCell; + +impl SLock for SRefCell { + type Lock = RefCell; +} + +impl LockLike for RefCell { + type LockGuard<'a> = RefMut<'a, T> where T: 'a; + + #[inline] + fn new(val: T) -> Self { + RefCell::new(val) + } + + #[inline] + fn into_inner(self) -> T { + self.into_inner() + } + + #[inline] + fn get_mut(&mut self) -> &mut T { + self.get_mut() + } + + #[inline] + fn try_lock(&self) -> Option> { + self.try_borrow_mut().ok() + } + + #[inline(always)] + #[track_caller] + fn lock(&self) -> RefMut<'_, T> { + self.borrow_mut() + } +} + +#[derive(Copy, Clone, Default)] +pub struct SMutex; + +impl SLock for SMutex { + type Lock = Mutex; +} + +impl LockLike for Mutex { + type LockGuard<'a> = MutexGuard<'a, T> where T: 'a; + + #[inline] + fn new(val: T) -> Self { + Mutex::new(val) + } + + #[inline] + fn into_inner(self) -> T { + self.into_inner().unwrap() + } + + #[inline] + fn get_mut(&mut self) -> &mut T { + self.get_mut().unwrap() + } + + #[inline] + fn try_lock(&self) -> Option> { + self.try_lock().ok() + } + + #[inline(always)] + #[track_caller] + fn lock(&self) -> MutexGuard<'_, T> { + self.lock().unwrap_or_else(|e| { + self.clear_poison(); + e.into_inner() + }) + } +} + pub struct MappedReadGuard<'a, T: ?Sized> { raw: &'a RwLockRaw, data: *const T, @@ -1138,7 +1235,7 @@ impl Clone for RwLock { pub struct WorkerLocal { single_thread: bool, inner: Option, - mt_inner: Option>, + mt_inner: Option>, } impl WorkerLocal { @@ -1152,20 +1249,10 @@ impl WorkerLocal { WorkerLocal { single_thread: false, inner: None, - mt_inner: Some(rayon_core::WorkerLocal::new(f)), + mt_inner: Some(worker_local::WorkerLocal::new(f)), } } } - - /// Returns the worker-local value for each thread - #[inline] - pub fn into_inner(self) -> Vec { - if self.single_thread { - vec![self.inner.unwrap()] - } else { - self.mt_inner.unwrap().into_inner() - } - } } impl Deref for WorkerLocal { @@ -1185,6 +1272,7 @@ impl Deref for WorkerLocal { unsafe impl std::marker::Sync for WorkerLocal {} use std::thread; +pub use worker_local::Registry; /// A type which only allows its inner value to be used in one thread. /// It will panic if it is used on multiple threads. diff --git a/compiler/rustc_data_structures/src/sync/worker_local.rs b/compiler/rustc_data_structures/src/sync/worker_local.rs index bfb04ba8a73f4..c3568c9594338 100644 --- a/compiler/rustc_data_structures/src/sync/worker_local.rs +++ b/compiler/rustc_data_structures/src/sync/worker_local.rs @@ -5,7 +5,6 @@ use std::ops::Deref; use std::ptr; use std::sync::Arc; -#[cfg(parallel_compiler)] use {crate::cold_path, crate::sync::CacheAligned}; /// A pointer to the `RegistryData` which uniquely identifies a registry. @@ -21,7 +20,6 @@ impl RegistryId { /// /// Note that there's a race possible where the identifer in `THREAD_DATA` could be reused /// so this can succeed from a different registry. - #[cfg(parallel_compiler)] fn verify(self) -> usize { let (id, index) = THREAD_DATA.with(|data| (data.registry_id.get(), data.index.get())); @@ -33,13 +31,14 @@ impl RegistryId { } } +#[derive(Debug)] struct RegistryData { thread_limit: usize, threads: Lock, } /// Represents a list of threads which can access worker locals. -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct Registry(Arc); thread_local! { @@ -105,12 +104,9 @@ impl Registry { /// Holds worker local values for each possible thread in a registry. You can only access the /// worker local value through the `Deref` impl on the registry associated with the thread it was /// created on. It will panic otherwise. -pub struct WorkerLocal { - #[cfg(not(parallel_compiler))] - local: T, - #[cfg(parallel_compiler)] +#[derive(Debug)] +pub(crate) struct WorkerLocal { locals: Box<[CacheAligned]>, - #[cfg(parallel_compiler)] registry: Registry, } @@ -118,7 +114,6 @@ pub struct WorkerLocal { // or it will panic for threads without an associated local. So there isn't a need for `T` to do // it's own synchronization. The `verify` method on `RegistryId` has an issue where the the id // can be reused, but `WorkerLocal` has a reference to `Registry` which will prevent any reuse. -#[cfg(parallel_compiler)] unsafe impl Sync for WorkerLocal {} impl WorkerLocal { @@ -126,52 +121,18 @@ impl WorkerLocal { /// value this worker local should take for each thread in the registry. #[inline] pub fn new T>(mut initial: F) -> WorkerLocal { - #[cfg(parallel_compiler)] - { - let registry = Registry::current(); - WorkerLocal { - locals: (0..registry.0.thread_limit).map(|i| CacheAligned(initial(i))).collect(), - registry, - } - } - #[cfg(not(parallel_compiler))] - { - WorkerLocal { local: initial(0) } - } - } - - /// Returns the worker-local values for each thread - #[inline] - pub fn into_inner(self) -> impl Iterator { - #[cfg(parallel_compiler)] - { - self.locals.into_vec().into_iter().map(|local| local.0) - } - #[cfg(not(parallel_compiler))] - { - std::iter::once(self.local) + let registry = Registry::current(); + WorkerLocal { + locals: (0..registry.0.thread_limit).map(|i| CacheAligned(initial(i))).collect(), + registry, } } } -impl WorkerLocal> { - /// Joins the elements of all the worker locals into one Vec - pub fn join(self) -> Vec { - self.into_inner().into_iter().flat_map(|v| v).collect() - } -} - impl Deref for WorkerLocal { type Target = T; #[inline(always)] - #[cfg(not(parallel_compiler))] - fn deref(&self) -> &T { - &self.local - } - - #[inline(always)] - #[cfg(parallel_compiler)] fn deref(&self) -> &T { // This is safe because `verify` will only return values less than // `self.registry.thread_limit` which is the size of the `self.locals` array. diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index 647f4826876da..1ab378cbfe998 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -10,7 +10,7 @@ use crate::ty::TyCtxt; use field_offset::FieldOffset; use measureme::StringId; use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::AtomicU64; +use rustc_data_structures::sync::{AtomicU64, Lrc, SLock, SMutex, SRefCell, WorkerLocal}; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::hir_id::OwnerId; @@ -20,6 +20,7 @@ pub(crate) use rustc_query_system::query::QueryJobId; use rustc_query_system::query::*; use rustc_query_system::HandleCycleError; use rustc_span::{Span, DUMMY_SP}; +use std::intrinsics::likely; use std::ops::Deref; pub struct QueryKeyStringCache { @@ -77,13 +78,15 @@ pub struct QuerySystemFns<'tcx> { ), pub try_mark_green: fn(tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool, } - pub struct QuerySystem<'tcx> { pub states: QueryStates<'tcx>, pub arenas: QueryArenas<'tcx>, - pub caches: QueryCaches<'tcx>, pub dynamic_queries: DynamicQueries<'tcx>, + pub single_thread: bool, + pub single_caches: QueryCaches<'tcx, SRefCell>, + pub parallel_caches: QueryCaches<'tcx, SMutex>, + /// This provides access to the incremental compilation on-disk cache for query results. /// Do not access this directly. It is only meant to be used by /// `DepGraph::try_mark_green()` and the query infrastructure. @@ -321,7 +324,7 @@ macro_rules! define_callbacks { use super::*; $( - pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>>>::Cache; + pub type $name<'tcx, L> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>, L>>::Cache; )* } @@ -374,21 +377,31 @@ macro_rules! define_callbacks { } #[derive(Default)] - pub struct QueryCaches<'tcx> { - $($(#[$attr])* pub $name: query_storage::$name<'tcx>,)* + pub struct QueryCaches<'tcx, L: SLock> { + $($(#[$attr])* pub $name: query_storage::$name<'tcx, L>,)* } impl<'tcx> TyCtxtEnsure<'tcx> { $($(#[$attr])* #[inline(always)] pub fn $name(self, key: query_helper_param_ty!($($K)*)) { - query_ensure( - self.tcx, - self.tcx.query_system.fns.engine.$name, - &self.tcx.query_system.caches.$name, - key.into_query_param(), - false, - ); + if likely(self.tcx.query_system.single_thread) { + query_ensure( + self.tcx, + self.tcx.query_system.fns.engine.$name, + &self.tcx.query_system.single_caches.$name, + key.into_query_param(), + false, + ) + } else { + query_ensure( + self.tcx, + self.tcx.query_system.fns.engine.$name, + &self.tcx.query_system.parallel_caches.$name, + key.into_query_param(), + false, + ) + }; })* } @@ -396,13 +409,23 @@ macro_rules! define_callbacks { $($(#[$attr])* #[inline(always)] pub fn $name(self, key: query_helper_param_ty!($($K)*)) { - query_ensure( - self.tcx, - self.tcx.query_system.fns.engine.$name, - &self.tcx.query_system.caches.$name, - key.into_query_param(), - true, - ); + if likely(self.tcx.query_system.single_thread) { + query_ensure( + self.tcx, + self.tcx.query_system.fns.engine.$name, + &self.tcx.query_system.single_caches.$name, + key.into_query_param(), + true, + ); + } else { + query_ensure( + self.tcx, + self.tcx.query_system.fns.engine.$name, + &self.tcx.query_system.parallel_caches.$name, + key.into_query_param(), + true, + ); + }; })* } @@ -421,13 +444,23 @@ macro_rules! define_callbacks { #[inline(always)] pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> $V { - restore::<$V>(query_get_at( - self.tcx, - self.tcx.query_system.fns.engine.$name, - &self.tcx.query_system.caches.$name, - self.span, - key.into_query_param(), - )) + if likely(self.tcx.query_system.single_thread) { + restore::<$V>(query_get_at( + self.tcx, + self.tcx.query_system.fns.engine.$name, + &self.tcx.query_system.single_caches.$name, + self.span, + key.into_query_param(), + )) + } else { + restore::<$V>(query_get_at( + self.tcx, + self.tcx.query_system.fns.engine.$name, + &self.tcx.query_system.parallel_caches.$name, + self.span, + key.into_query_param(), + )) + } })* } @@ -523,38 +556,75 @@ macro_rules! define_feedable { let tcx = self.tcx; let erased = query_provided_to_value::$name(tcx, value); let value = restore::<$V>(erased); - let cache = &tcx.query_system.caches.$name; - - let hasher: Option, &_) -> _> = hash_result!([$($modifiers)*]); - match try_get_cached(tcx, cache, &key) { - Some(old) => { - let old = restore::<$V>(old); - if let Some(hasher) = hasher { - let (value_hash, old_hash): (Fingerprint, Fingerprint) = tcx.with_stable_hashing_context(|mut hcx| - (hasher(&mut hcx, &value), hasher(&mut hcx, &old)) + if likely(tcx.query_system.single_thread) { + let cache = &tcx.query_system.single_caches.$name; + + let hasher: Option, &_) -> _> = hash_result!([$($modifiers)*]); + match try_get_cached(tcx, cache, &key) { + Some(old) => { + let old = restore::<$V>(old); + if let Some(hasher) = hasher { + let (value_hash, old_hash): (Fingerprint, Fingerprint) = tcx.with_stable_hashing_context(|mut hcx| + (hasher(&mut hcx, &value), hasher(&mut hcx, &old)) + ); + assert_eq!( + old_hash, value_hash, + "Trying to feed an already recorded value for query {} key={key:?}:\nold value: {old:?}\nnew value: {value:?}", + stringify!($name), + ) + } else { + bug!( + "Trying to feed an already recorded value for query {} key={key:?}:\nold value: {old:?}\nnew value: {value:?}", + stringify!($name), + ) + } + } + None => { + let dep_node = dep_graph::DepNode::construct(tcx, dep_graph::DepKind::$name, &key); + let dep_node_index = tcx.dep_graph.with_feed_task( + dep_node, + tcx, + key, + &value, + hash_result!([$($modifiers)*]), ); - assert_eq!( - old_hash, value_hash, - "Trying to feed an already recorded value for query {} key={key:?}:\nold value: {old:?}\nnew value: {value:?}", - stringify!($name), - ) - } else { - bug!( - "Trying to feed an already recorded value for query {} key={key:?}:\nold value: {old:?}\nnew value: {value:?}", - stringify!($name), - ) + cache.complete(key, erased, dep_node_index); } } - None => { - let dep_node = dep_graph::DepNode::construct(tcx, dep_graph::DepKind::$name, &key); - let dep_node_index = tcx.dep_graph.with_feed_task( - dep_node, - tcx, - key, - &value, - hash_result!([$($modifiers)*]), - ); - cache.complete(key, erased, dep_node_index); + } else { + let cache = &tcx.query_system.parallel_caches.$name; + + let hasher: Option, &_) -> _> = hash_result!([$($modifiers)*]); + match try_get_cached(tcx, cache, &key) { + Some(old) => { + let old = restore::<$V>(old); + if let Some(hasher) = hasher { + let (value_hash, old_hash): (Fingerprint, Fingerprint) = tcx.with_stable_hashing_context(|mut hcx| + (hasher(&mut hcx, &value), hasher(&mut hcx, &old)) + ); + assert_eq!( + old_hash, value_hash, + "Trying to feed an already recorded value for query {} key={key:?}:\nold value: {old:?}\nnew value: {value:?}", + stringify!($name), + ) + } else { + bug!( + "Trying to feed an already recorded value for query {} key={key:?}:\nold value: {old:?}\nnew value: {value:?}", + stringify!($name), + ) + } + } + None => { + let dep_node = dep_graph::DepNode::construct(tcx, dep_graph::DepKind::$name, &key); + let dep_node_index = tcx.dep_graph.with_feed_task( + dep_node, + tcx, + key, + &value, + hash_result!([$($modifiers)*]), + ); + cache.complete(key, erased, dep_node_index); + } } } } diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index b76734dd07294..893149d64560d 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -7,6 +7,7 @@ #![feature(min_specialization)] #![feature(never_type)] #![feature(rustc_attrs)] +#![feature(core_intrinsics)] #![recursion_limit = "256"] #![allow(rustc::potential_query_instability, unused_parens)] #![deny(rustc::untranslatable_diagnostic)] @@ -18,7 +19,7 @@ extern crate rustc_middle; use crate::plumbing::{encode_all_query_results, try_mark_green}; use field_offset::offset_of; use rustc_data_structures::stable_hasher::HashStable; -use rustc_data_structures::sync::AtomicU64; +use rustc_data_structures::sync::{SLock, SMutex, SRefCell}; use rustc_middle::arena::Arena; use rustc_middle::dep_graph::DepNodeIndex; use rustc_middle::dep_graph::{self, DepKind, DepKindStruct}; @@ -31,7 +32,7 @@ use rustc_middle::query::{ DynamicQueries, ExternProviders, Providers, QueryCaches, QueryEngine, QueryStates, }; use rustc_middle::ty::TyCtxt; -use rustc_query_system::dep_graph::SerializedDepNodeIndex; +use rustc_query_system::dep_graph::{DepNodeIndex, SerializedDepNodeIndex}; use rustc_query_system::ich::StableHashingContext; use rustc_query_system::query::{ get_query_incr, get_query_non_incr, HashResult, QueryCache, QueryConfig, QueryInfo, QueryMap, @@ -40,6 +41,7 @@ use rustc_query_system::query::{ use rustc_query_system::HandleCycleError; use rustc_query_system::Value; use rustc_span::Span; +use std::intrinsics::likely; #[macro_use] mod plumbing; diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index 79d8abc4b694c..a670084807ef8 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -28,6 +28,7 @@ use rustc_serialize::Decodable; use rustc_serialize::Encodable; use rustc_session::Limit; use rustc_span::def_id::LOCAL_CRATE; +use std::intrinsics::likely; use std::num::NonZeroU64; use thin_vec::ThinVec; @@ -354,19 +355,35 @@ pub(crate) fn encode_query_results<'a, 'tcx, Q>( qcx.profiler().verbose_generic_activity_with_arg("encode_query_results_for", query.name()); assert!(query.query_state(qcx).all_inactive()); - let cache = query.query_cache(qcx); - cache.iter(&mut |key, value, dep_node| { - if query.cache_on_disk(qcx.tcx, &key) { - let dep_node = SerializedDepNodeIndex::new(dep_node.index()); + if likely(query.single_thread(qcx)) { + let cache = query.single_query_cache(qcx); + cache.iter(&mut |key, value, dep_node| { + if query.cache_on_disk(qcx.tcx, &key) { + let dep_node = SerializedDepNodeIndex::new(dep_node.index()); + + // Record position of the cache entry. + query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position()))); + + // Encode the type check tables with the `SerializedDepNodeIndex` + // as tag. + encoder.encode_tagged(dep_node, &Q::restore(*value)); + } + }); + } else { + let cache = query.parallel_query_cache(qcx); + cache.iter(&mut |key, value, dep_node| { + if query.cache_on_disk(qcx.tcx, &key) { + let dep_node = SerializedDepNodeIndex::new(dep_node.index()); - // Record position of the cache entry. - query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position()))); + // Record position of the cache entry. + query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position()))); - // Encode the type check tables with the `SerializedDepNodeIndex` - // as tag. - encoder.encode_tagged(dep_node, &Q::restore(*value)); - } - }); + // Encode the type check tables with the `SerializedDepNodeIndex` + // as tag. + encoder.encode_tagged(dep_node, &Q::restore(*value)); + } + }); + } } fn try_load_from_on_disk_cache<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode) @@ -613,6 +630,42 @@ macro_rules! define_queries { }, hash_result: hash_result!([$($modifiers)*][query_values::$name<'tcx>]), format_value: |value| format!("{:?}", restore::>(*value)), + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + } } )* @@ -754,12 +807,21 @@ macro_rules! define_queries { ) }, alloc_self_profile_query_strings: |tcx, string_cache| { - $crate::profiling_support::alloc_self_profile_query_strings_for_query_cache( - tcx, - stringify!($name), - &tcx.query_system.caches.$name, - string_cache, - ) + if tcx.query_system.single_thread { + $crate::profiling_support::alloc_self_profile_query_strings_for_query_cache( + tcx, + stringify!($name), + &tcx.query_system.single_caches.$name, + string_cache, + ) + } else { + $crate::profiling_support::alloc_self_profile_query_strings_for_query_cache( + tcx, + stringify!($name), + &tcx.query_system.parallel_caches.$name, + string_cache, + ) + } }, encode_query_results: expand_if_cached!([$($modifiers)*], |tcx, encoder, query_result_index| $crate::plumbing::encode_query_results::>( diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs index d17711d13630d..adc2808fd1d01 100644 --- a/compiler/rustc_query_system/src/query/caches.rs +++ b/compiler/rustc_query_system/src/query/caches.rs @@ -2,17 +2,17 @@ use crate::dep_graph::DepNodeIndex; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sharded; -use rustc_data_structures::sharded::Sharded; -use rustc_data_structures::sync::Lock; +use rustc_data_structures::sync::{LockLike, SLock}; use rustc_index::{Idx, IndexVec}; use std::fmt::Debug; use std::hash::Hash; use std::marker::PhantomData; -pub trait CacheSelector<'tcx, V> { +pub trait CacheSelector<'tcx, V, L> { type Cache where - V: Copy; + V: Copy, + L: SLock; } pub trait QueryCache: Sized { @@ -29,23 +29,23 @@ pub trait QueryCache: Sized { pub struct DefaultCacheSelector(PhantomData); -impl<'tcx, K: Eq + Hash, V: 'tcx> CacheSelector<'tcx, V> for DefaultCacheSelector { - type Cache = DefaultCache +impl<'tcx, K: Eq + Hash, V: 'tcx, L: SLock> CacheSelector<'tcx, V, L> for DefaultCacheSelector { + type Cache = DefaultCache where V: Copy; } -pub struct DefaultCache { - cache: Sharded>, +pub struct DefaultCache { + cache: L::Lock>, } -impl Default for DefaultCache { +impl Default for DefaultCache { fn default() -> Self { - DefaultCache { cache: Default::default() } + DefaultCache { cache: L::Lock::new(FxHashMap::default()) } } } -impl QueryCache for DefaultCache +impl QueryCache for DefaultCache where K: Eq + Hash + Copy + Debug, V: Copy, @@ -56,48 +56,50 @@ where #[inline(always)] fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> { let key_hash = sharded::make_hash(key); - self.cache.with_get_shard_by_hash(key_hash, |lock| { - let result = lock.raw_entry().from_key_hashed_nocheck(key_hash, key); - if let Some((_, value)) = result { Some(*value) } else { None } - }) + + let lock = self.cache.lock(); + + let result = lock.raw_entry().from_key_hashed_nocheck(key_hash, key); + + if let Some((_, value)) = result { Some(*value) } else { None } } #[inline] fn complete(&self, key: K, value: V, index: DepNodeIndex) { + let mut lock = self.cache.lock(); + // We may be overwriting another value. This is all right, since the dep-graph // will check that the fingerprint matches. - self.cache.with_get_shard_by_value(&key, |cache| cache.insert(key, (value, index))); + lock.insert(key, (value, index)); } fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) { - let shards = self.cache.lock_shards(); - for shard in shards.iter() { - for (k, v) in shard.iter() { - f(k, &v.0, v.1); - } + let shards = self.cache.lock(); + for (k, v) in shards.iter() { + f(k, &v.0, v.1); } } } pub struct SingleCacheSelector; -impl<'tcx, V: 'tcx> CacheSelector<'tcx, V> for SingleCacheSelector { - type Cache = SingleCache +impl<'tcx, V: 'tcx, L: SLock> CacheSelector<'tcx, V, L> for SingleCacheSelector { + type Cache = SingleCache where V: Copy; } -pub struct SingleCache { - cache: Lock>, +pub struct SingleCache { + cache: L::Lock>, } -impl Default for SingleCache { +impl Default for SingleCache { fn default() -> Self { - SingleCache { cache: Lock::new(None) } + SingleCache { cache: L::Lock::new(None) } } } -impl QueryCache for SingleCache +impl QueryCache for SingleCache where V: Copy, { @@ -106,12 +108,12 @@ where #[inline(always)] fn lookup(&self, _key: &()) -> Option<(V, DepNodeIndex)> { - self.cache.with_lock(|cache| *cache) + *self.cache.lock() } #[inline] fn complete(&self, _key: (), value: V, index: DepNodeIndex) { - self.cache.with_lock(|cache| *cache = Some((value, index))); + *self.cache.lock() = Some((value, index)); } fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) { @@ -123,23 +125,23 @@ where pub struct VecCacheSelector(PhantomData); -impl<'tcx, K: Idx, V: 'tcx> CacheSelector<'tcx, V> for VecCacheSelector { - type Cache = VecCache +impl<'tcx, K: Idx, V: 'tcx, L: SLock> CacheSelector<'tcx, V, L> for VecCacheSelector { + type Cache = VecCache where V: Copy; } -pub struct VecCache { - cache: Sharded>>, +pub struct VecCache { + cache: L::Lock>>, } -impl Default for VecCache { +impl Default for VecCache { fn default() -> Self { - VecCache { cache: Default::default() } + VecCache { cache: L::Lock::new(IndexVec::default()) } } } -impl QueryCache for VecCache +impl QueryCache for VecCache where K: Eq + Idx + Copy + Debug, V: Copy, @@ -149,25 +151,23 @@ where #[inline(always)] fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> { - self.cache.with_get_shard_by_hash(key.index() as u64, |lock| { - if let Some(Some(value)) = lock.get(*key) { Some(*value) } else { None } - }) + let lock = self.cache.lock(); + + if let Some(Some(value)) = lock.get(*key) { Some(*value) } else { None } } #[inline] fn complete(&self, key: K, value: V, index: DepNodeIndex) { - self.cache.with_get_shard_by_hash(key.index() as u64, |lock| { - lock.insert(key, (value, index)); - }) + let mut lock = self.cache.lock(); + + lock.insert(key, (value, index)); } fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) { - let shards = self.cache.lock_shards(); - for shard in shards.iter() { - for (k, v) in shard.iter_enumerated() { - if let Some(v) = v { - f(&k, &v.0, v.1); - } + let shards = self.cache.lock(); + for (k, v) in shards.iter_enumerated() { + if let Some(v) = v { + f(&k, &v.0, v.1); } } } diff --git a/compiler/rustc_query_system/src/query/config.rs b/compiler/rustc_query_system/src/query/config.rs index 7e47d70120544..5c0f5d5932702 100644 --- a/compiler/rustc_query_system/src/query/config.rs +++ b/compiler/rustc_query_system/src/query/config.rs @@ -1,6 +1,6 @@ //! Query configuration and description traits. -use crate::dep_graph::{DepNode, DepNodeParams, SerializedDepNodeIndex}; +use crate::dep_graph::{DepNode, DepNodeIndex, DepNodeParams, SerializedDepNodeIndex}; use crate::error::HandleCycleError; use crate::ich::StableHashingContext; use crate::query::caches::QueryCache; @@ -8,6 +8,7 @@ use crate::query::DepNodeIndex; use crate::query::{QueryContext, QueryInfo, QueryState}; use rustc_data_structures::fingerprint::Fingerprint; +use rustc_data_structures::sync::{SLock, SMutex, SRefCell}; use std::fmt::Debug; use std::hash::Hash; @@ -21,17 +22,24 @@ pub trait QueryConfig: Copy { type Key: DepNodeParams + Eq + Hash + Copy + Debug; type Value: Copy; - type Cache: QueryCache; + type Cache: QueryCache; + + fn single_thread(self, tcx: Qcx) -> bool; fn format_value(self) -> fn(&Self::Value) -> String; - // Don't use this method to access query results, instead use the methods on TyCtxt - fn query_state<'a>(self, tcx: Qcx) -> &'a QueryState + fn look_up(self, tcx: Qcx, key: &Self::Key) -> Option<(Self::Value, DepNodeIndex)>; + + fn single_query_cache<'a>(self, tcx: Qcx) -> &'a Self::Cache + where + Qcx: 'a; + + fn parallel_query_cache<'a>(self, tcx: Qcx) -> &'a Self::Cache where Qcx: 'a; // Don't use this method to access query results, instead use the methods on TyCtxt - fn query_cache<'a>(self, tcx: Qcx) -> &'a Self::Cache + fn query_state<'a>(self, tcx: Qcx) -> &'a QueryState where Qcx: 'a; diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index bdd8fbbad7e9d..cafa9f6d60437 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -26,6 +26,7 @@ use std::cell::Cell; use std::collections::hash_map::Entry; use std::fmt::Debug; use std::hash::Hash; +use std::intrinsics::likely; use std::mem; use std::ops::DerefMut; use thin_vec::ThinVec; @@ -268,7 +269,7 @@ where match result { Ok(()) => { - let Some((v, index)) = query.query_cache(qcx).lookup(&key) else { + let Some((v, index)) = query.look_up(qcx, &key) else { cold_path(|| panic!("value must be in cache after waiting")) }; @@ -304,7 +305,7 @@ where // executing, but another thread may have already completed the query and stores it result // in the query cache. if cfg!(parallel_compiler) && qcx.dep_context().sess().threads() > 1 { - if let Some((value, index)) = query.query_cache(qcx).lookup(&key) { + if let Some((value, index)) = query.look_up(qcx, &key) { qcx.dep_context().profiler().query_cache_hit(index.into()); return (value, Some(index)); } @@ -389,13 +390,12 @@ where execute_job_non_incr(query, qcx, key, id) }; - let cache = query.query_cache(qcx); if query.feedable() { // We should not compute queries that also got a value via feeding. // This can't happen, as query feeding adds the very dependencies to the fed query // as its feeding query had. So if the fed query is red, so is its feeder, which will // get evaluated first, and re-feed the query. - if let Some((cached_result, _)) = cache.lookup(&key) { + if let Some((cached_result, _)) = query.look_up(qcx, &key) { let Some(hasher) = query.hash_result() else { panic!( "no_hash fed query later has its value computed.\n\ @@ -421,7 +421,12 @@ where ); } } - job_owner.complete(cache, result, dep_node_index); + + if likely(query.single_thread(qcx)) { + job_owner.complete(query.single_query_cache(qcx), result, dep_node_index) + } else { + job_owner.complete(query.parallel_query_cache(qcx), result, dep_node_index) + }; (result, Some(dep_node_index)) } @@ -824,7 +829,7 @@ pub fn force_query( { // We may be concurrently trying both execute and force a query. // Ensure that only one of them runs the query. - if let Some((_, index)) = query.query_cache(qcx).lookup(&key) { + if let Some((_, index)) = query.look_up(qcx, &key) { qcx.dep_context().profiler().query_cache_hit(index.into()); return; } diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 710c8b52194b4..e51fe601149db 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -1147,7 +1147,7 @@ impl Config { set(&mut config.use_lld, rust.use_lld); set(&mut config.lld_enabled, rust.lld); set(&mut config.llvm_tools_enabled, rust.llvm_tools); - config.rustc_parallel = rust.parallel_compiler.unwrap_or(false); + config.rustc_parallel = rust.parallel_compiler.unwrap_or(true); config.rustc_default_linker = rust.default_linker; config.musl_root = rust.musl_root.map(PathBuf::from); config.save_toolstates = rust.save_toolstates.map(PathBuf::from); From 64714b6739672e74de5c532da891c6bc6a4bee4f Mon Sep 17 00:00:00 2001 From: SparrowLii Date: Thu, 11 May 2023 19:56:26 +0800 Subject: [PATCH 08/11] close parallel compilation for Incremental --- compiler/rustc_data_structures/src/sync.rs | 30 ++++++++----------- compiler/rustc_driver_impl/src/lib.rs | 2 +- compiler/rustc_interface/src/interface.rs | 14 +++++++-- compiler/rustc_interface/src/lib.rs | 1 + compiler/rustc_interface/src/util.rs | 2 +- compiler/rustc_metadata/src/rmeta/encoder.rs | 2 +- compiler/rustc_query_system/src/query/job.rs | 9 ++++-- .../rustc_query_system/src/query/plumbing.rs | 2 +- compiler/rustc_session/src/options.rs | 2 +- src/librustdoc/lib.rs | 2 +- 10 files changed, 39 insertions(+), 27 deletions(-) diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs index a34a3231d4f66..4878c828f4747 100644 --- a/compiler/rustc_data_structures/src/sync.rs +++ b/compiler/rustc_data_structures/src/sync.rs @@ -53,14 +53,13 @@ mod worker_local; pub use std::sync::atomic::Ordering; pub use std::sync::atomic::Ordering::SeqCst; -use std::sync::{Mutex, MutexGuard}; pub use vec::{AppendOnlyIndexVec, AppendOnlyVec}; mod vec; use parking_lot::lock_api::RawMutex as _; use parking_lot::lock_api::RawRwLock as _; -use parking_lot::{RawMutex, RawRwLock}; +use parking_lot::{Mutex, MutexGuard, RawMutex, RawRwLock}; mod mode { use super::Ordering; @@ -644,7 +643,7 @@ impl Lock { } } - #[inline(never)] + #[inline] fn lock_raw(&self) { if likely(self.single_thread) { assert!(!self.borrow.replace(true)); @@ -660,7 +659,7 @@ impl Lock { LockGuard { lock: &self, marker: PhantomData } } - #[inline(never)] + #[inline] pub(crate) fn with_mt_lock R, R>(&self, f: F) -> R { unsafe { self.mutex.lock(); @@ -683,7 +682,7 @@ impl Lock { } } - #[inline(never)] + #[inline] fn with_mt_borrow R, R>(&self, f: F) -> R { unsafe { self.mutex.lock(); @@ -751,7 +750,7 @@ impl DerefMut for LockGuard<'_, T> { } } -#[inline(never)] +#[inline] fn unlock_mt(guard: &mut LockGuard<'_, T>) { unsafe { guard.lock.mutex.unlock() } } @@ -842,26 +841,23 @@ impl LockLike for Mutex { #[inline] fn into_inner(self) -> T { - self.into_inner().unwrap() + self.into_inner() } #[inline] fn get_mut(&mut self) -> &mut T { - self.get_mut().unwrap() + self.get_mut() } #[inline] fn try_lock(&self) -> Option> { - self.try_lock().ok() + self.try_lock() } #[inline(always)] #[track_caller] fn lock(&self) -> MutexGuard<'_, T> { - self.lock().unwrap_or_else(|e| { - self.clear_poison(); - e.into_inner() - }) + self.lock() } } @@ -1095,7 +1091,7 @@ impl RwLock { self.data.get_mut() } - #[inline(never)] + #[inline] fn mt_read(&self) -> ReadGuard<'_, T> { self.raw.raw.lock_shared(); ReadGuard { rwlock: self, marker: PhantomData } @@ -1113,7 +1109,7 @@ impl RwLock { } } - #[inline(never)] + #[inline] fn with_mt_read_lock R, R>(&self, f: F) -> R { self.raw.raw.lock_shared(); let r = unsafe { f(&*self.data.get()) }; @@ -1157,7 +1153,7 @@ impl RwLock { } } - #[inline(never)] + #[inline] fn mt_write(&self) -> WriteGuard<'_, T> { self.raw.raw.lock_exclusive(); WriteGuard { rwlock: self, marker: PhantomData } @@ -1173,7 +1169,7 @@ impl RwLock { } } - #[inline(never)] + #[inline] pub fn with_mt_write_lock R, R>(&self, f: F) -> R { self.raw.raw.lock_exclusive(); unsafe { diff --git a/compiler/rustc_driver_impl/src/lib.rs b/compiler/rustc_driver_impl/src/lib.rs index 9846d721616ac..6e7649587e12f 100644 --- a/compiler/rustc_driver_impl/src/lib.rs +++ b/compiler/rustc_driver_impl/src/lib.rs @@ -257,7 +257,7 @@ fn run_compiler( let sopts = config::build_session_options(&matches); // Set parallel mode before thread pool creation, which will create `Lock`s. - interface::set_parallel_mode(&sopts.unstable_opts); + interface::set_parallel_mode(&sopts.unstable_opts, &sopts.cg); if let Some(ref code) = matches.opt_str("explain") { handle_explain(diagnostics_registry(), code, sopts.error_format); diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs index 6417a5cd722cf..7f99245bbe686 100644 --- a/compiler/rustc_interface/src/interface.rs +++ b/compiler/rustc_interface/src/interface.rs @@ -61,8 +61,18 @@ impl Compiler { } #[allow(rustc::bad_opt_access)] -pub fn set_parallel_mode(sopts: &config::UnstableOptions) { - rustc_data_structures::sync::set(sopts.threads > 1); +pub fn set_parallel_mode(sopts1: &config::UnstableOptions, sopts2: &config::CodegenOptions) { + let parallel = if sopts1.threads <= 1 { + false + } else { + if let Some(path) = &sopts2.incremental { + if matches!(std::fs::try_exists(PathBuf::from(path)), Ok(false)) { true } else { false } + } else { + true + } + }; + + rustc_data_structures::sync::set(parallel); } /// Converts strings provided as `--cfg [cfgspec]` into a `crate_cfg`. diff --git a/compiler/rustc_interface/src/lib.rs b/compiler/rustc_interface/src/lib.rs index 51bd8381e93d7..9ca94a52fddc3 100644 --- a/compiler/rustc_interface/src/lib.rs +++ b/compiler/rustc_interface/src/lib.rs @@ -4,6 +4,7 @@ #![feature(thread_spawn_unchecked)] #![feature(lazy_cell)] #![feature(try_blocks)] +#![feature(fs_try_exists)] #![recursion_limit = "256"] #![allow(rustc::potential_query_instability)] #![deny(rustc::untranslatable_diagnostic)] diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs index d0336063024c3..08e218ea49793 100644 --- a/compiler/rustc_interface/src/util.rs +++ b/compiler/rustc_interface/src/util.rs @@ -134,7 +134,7 @@ pub(crate) fn run_in_thread_pool_with_globals R + Send, R: Send>( f: F, ) -> R { #[cfg(parallel_compiler)] - if _threads > 1 { + if rustc_data_structures::sync::active() { return run_in_threads_pool_with_globals(edition, _threads, f); } // The "thread pool" is a single spawned thread in the non-parallel diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index 79eb48a1a3155..ed3a9637eab43 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2233,7 +2233,7 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path) { join( || encode_metadata_impl(tcx, path), || { - if tcx.sess.threads() == 1 { + if !rustc_data_structures::sync::active() { return; } // Prefetch some queries used by metadata encoding. diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs index da12f6c3f7b9d..0dc103c015bb7 100644 --- a/compiler/rustc_query_system/src/query/job.rs +++ b/compiler/rustc_query_system/src/query/job.rs @@ -612,7 +612,7 @@ pub(crate) fn report_cycle<'a, D: DepKind>( } pub fn print_query_stack( - qcx: Qcx, + _qcx: Qcx, mut current_query: Option, handler: &Handler, num_frames: Option, @@ -621,7 +621,12 @@ pub fn print_query_stack( // a panic hook, which means that the global `Handler` may be in a weird // state if it was responsible for triggering the panic. let mut i = 0; - let query_map = qcx.try_collect_active_jobs(); + + #[cfg(not(parallel_compiler))] + let query_map = _qcx.try_collect_active_jobs(); + + #[cfg(parallel_compiler)] + let query_map: Option> = None; while let Some(query) = current_query { if Some(i) == num_frames { diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index cafa9f6d60437..18999010adf5d 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -304,7 +304,7 @@ where // re-executing the query since `try_start` only checks that the query is not currently // executing, but another thread may have already completed the query and stores it result // in the query cache. - if cfg!(parallel_compiler) && qcx.dep_context().sess().threads() > 1 { + if cfg!(parallel_compiler) && rustc_data_structures::sync::active() { if let Some((value, index)) = query.look_up(qcx, &key) { qcx.dep_context().profiler().query_cache_hit(index.into()); return (value, Some(index)); diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 5976b9aa3e74a..17d9a95a10da2 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1748,7 +1748,7 @@ options! { /// in the future. Note that -Zthreads=0 is the way to get /// the num_cpus behavior. #[rustc_lint_opt_deny_field_access("use `Session::threads` instead of this field")] - threads: usize = (1, parse_threads, [UNTRACKED], + threads: usize = (2, parse_threads, [UNTRACKED], "use a thread pool with N threads"), time_llvm_passes: bool = (false, parse_bool, [UNTRACKED], "measure time of each LLVM pass (default: no)"), diff --git a/src/librustdoc/lib.rs b/src/librustdoc/lib.rs index 369de24bffeb3..d6964b9cf0709 100644 --- a/src/librustdoc/lib.rs +++ b/src/librustdoc/lib.rs @@ -738,7 +738,7 @@ fn main_args(at_args: &[String]) -> MainResult { }; // Set parallel mode before error handler creation, which will create `Lock`s. - interface::set_parallel_mode(&options.unstable_opts); + interface::set_parallel_mode(&options.unstable_opts, &options.codegen_options); let diag = core::new_handler( options.error_format, From 0796d1bdbef2c3d76e3e8d46d26adec04105772c Mon Sep 17 00:00:00 2001 From: SparrowLii Date: Wed, 17 May 2023 15:42:37 +0800 Subject: [PATCH 09/11] specialize `Sharded` --- compiler/rustc_data_structures/src/sharded.rs | 215 +++++++++++++++--- compiler/rustc_data_structures/src/sync.rs | 18 -- compiler/rustc_middle/src/query/plumbing.rs | 20 +- compiler/rustc_middle/src/ty/context.rs | 42 ++-- compiler/rustc_query_impl/src/lib.rs | 4 +- compiler/rustc_query_impl/src/plumbing.rs | 13 ++ .../rustc_query_system/src/dep_graph/graph.rs | 12 +- .../rustc_query_system/src/query/caches.rs | 75 +++--- .../rustc_query_system/src/query/config.rs | 8 +- .../rustc_query_system/src/query/plumbing.rs | 43 +++- 10 files changed, 325 insertions(+), 125 deletions(-) diff --git a/compiler/rustc_data_structures/src/sharded.rs b/compiler/rustc_data_structures/src/sharded.rs index 287a74190faa8..381892c057486 100644 --- a/compiler/rustc_data_structures/src/sharded.rs +++ b/compiler/rustc_data_structures/src/sharded.rs @@ -1,70 +1,236 @@ use crate::fx::{FxHashMap, FxHasher}; -use crate::sync::{Lock, LockGuard}; +use crate::sync::LockLike; +use parking_lot::{Mutex, MutexGuard}; use std::borrow::Borrow; +use std::cell::{RefCell, RefMut}; use std::collections::hash_map::RawEntryMut; use std::hash::{Hash, Hasher}; +use std::mem; -const SHARD_BITS: usize = 0; +pub trait Shard { + type Impl: ShardImpl; +} -pub const SHARDS: usize = 1 << SHARD_BITS; +pub trait ShardImpl { + type Lock: LockLike; + + fn new(value: impl FnMut() -> T) -> Self; + + fn get_shard_by_value(&self, _val: &K) -> &Self::Lock; + + fn get_shard_by_hash(&self, _hash: u64) -> &Self::Lock; + + fn lock_shards(&self) -> Vec<>::LockGuard<'_>>; + + fn try_lock_shards(&self) -> Option>::LockGuard<'_>>>; +} + +#[derive(Default)] +pub struct SingleShard; + +impl Shard for SingleShard { + type Impl = SingleShardImpl; +} /// An array of cache-line aligned inner locked structures with convenience methods. -pub struct Sharded { - shard: Lock, +pub struct SingleShardImpl { + shard: RefCell, +} + +impl Default for SingleShardImpl { + #[inline] + fn default() -> Self { + Self { shard: RefCell::new(T::default()) } + } +} + +impl ShardImpl for SingleShardImpl { + type Lock = RefCell; + + #[inline] + fn new(mut value: impl FnMut() -> T) -> Self { + SingleShardImpl { shard: RefCell::new(value()) } + } + + #[inline] + fn get_shard_by_value(&self, _val: &K) -> &RefCell { + &self.shard + } + + #[inline] + fn get_shard_by_hash(&self, _hash: u64) -> &RefCell { + &self.shard + } + + fn lock_shards(&self) -> Vec> { + vec![self.shard.lock()] + } + + fn try_lock_shards(&self) -> Option>> { + Some(vec![self.shard.try_lock()?]) + } +} + +const SHARD_BITS: usize = 5; + +pub const SHARDS: usize = 1 << SHARD_BITS; + +#[derive(Default)] +pub struct Sharded; + +impl Shard for Sharded { + type Impl = ShardedImpl; +} + +#[derive(Default)] +#[repr(align(64))] +pub struct CacheAligned(pub T); + +pub struct ShardedImpl { + shards: [CacheAligned>; SHARDS], } -impl Default for Sharded { +impl Default for ShardedImpl { #[inline] fn default() -> Self { Self::new(T::default) } } -impl Sharded { +impl ShardImpl for ShardedImpl { + type Lock = Mutex; + + #[inline] + fn new(mut value: impl FnMut() -> T) -> Self { + ShardedImpl { shards: [(); SHARDS].map(|()| CacheAligned(Mutex::new(value()))) } + } + + /// The shard is selected by hashing `val` with `FxHasher`. + #[inline] + fn get_shard_by_value(&self, val: &K) -> &Mutex { + self.get_shard_by_hash(make_hash(val)) + } + + #[inline] + fn get_shard_by_hash(&self, hash: u64) -> &Mutex { + &self.shards[get_shard_index_by_hash(hash)].0 + } + + fn lock_shards(&self) -> Vec> { + (0..SHARDS).map(|i| self.shards[i].0.lock()).collect() + } + + fn try_lock_shards(&self) -> Option>> { + (0..SHARDS).map(|i| self.shards[i].0.try_lock()).collect() + } +} + +pub struct DynSharded { + single_thread: bool, + single_shard: RefCell, + parallel_shard: ShardedImpl, +} + +// just for speed test +unsafe impl Sync for DynSharded {} + +impl Default for DynSharded { #[inline] + fn default() -> Self { + let single_thread = !crate::sync::active(); + DynSharded { + single_thread, + single_shard: RefCell::new(T::default()), + parallel_shard: ShardedImpl::default(), + } + } +} + +impl DynSharded { pub fn new(mut value: impl FnMut() -> T) -> Self { - Sharded { shard: Lock::new(value()) } + if !crate::sync::active() { + DynSharded { + single_thread: true, + single_shard: RefCell::new(value()), + parallel_shard: ShardedImpl::default(), + } + } else { + DynSharded { + single_thread: false, + single_shard: RefCell::new(T::default()), + parallel_shard: ShardedImpl::new(value), + } + } } /// The shard is selected by hashing `val` with `FxHasher`. #[inline] pub fn with_get_shard_by_value R, R>( &self, - _val: &K, + val: &K, f: F, ) -> R { - self.shard.with_lock(f) + if self.single_thread { + let mut lock = self.single_shard.borrow_mut(); + f(&mut *lock) + } else { + let mut lock = self.parallel_shard.get_shard_by_value(val).lock(); + f(&mut *lock) + } } #[inline] - pub fn with_get_shard_by_hash R, R>(&self, _hash: u64, f: F) -> R { - self.shard.with_lock(f) + pub fn with_get_shard_by_hash R, R>(&self, hash: u64, f: F) -> R { + if self.single_thread { + let mut lock = self.single_shard.borrow_mut(); + f(&mut *lock) + } else { + let mut lock = self.parallel_shard.get_shard_by_hash(hash).lock(); + f(&mut *lock) + } } #[inline] - pub fn get_shard_by_value(&self, _val: &K) -> &Lock { - &self.shard + pub fn with_lock_shards R, R>(&self, mut f: F) -> Vec { + if self.single_thread { + let mut lock = self.single_shard.borrow_mut(); + vec![f(&mut *lock)] + } else { + (0..SHARDS).map(|i| f(&mut *self.parallel_shard.shards[i].0.lock())).collect() + } } #[inline] - pub fn get_shard_by_hash(&self, _hash: u64) -> &Lock { - &self.shard + pub fn with_try_lock_shards R, R>(&self, mut f: F) -> Option> { + if self.single_thread { + let mut lock = self.single_shard.try_borrow_mut().ok()?; + Some(vec![f(&mut *lock)]) + } else { + (0..SHARDS) + .map(|i| { + let mut shard = self.parallel_shard.shards[i].0.try_lock()?; + Some(f(&mut *shard)) + }) + .collect() + } } - pub fn lock_shards(&self) -> Vec> { - vec![self.shard.lock()] + #[inline] + pub fn get_lock_by_value(&self, val: &K) -> &Mutex { + self.parallel_shard.get_shard_by_value(val) } - pub fn try_lock_shards(&self) -> Option>> { - Some(vec![self.shard.try_lock()?]) + #[inline] + pub fn get_borrow_by_value(&self, _val: &K) -> &RefCell { + &self.single_shard } } -pub type ShardedHashMap = Sharded>; +pub type ShardedHashMap = DynSharded>; impl ShardedHashMap { pub fn len(&self) -> usize { - self.lock_shards().iter().map(|shard| shard.len()).sum() + self.with_lock_shards(|shard| shard.len()).into_iter().sum() } } @@ -120,7 +286,6 @@ pub trait IntoPointer { impl ShardedHashMap { pub fn contains_pointer_to(&self, value: &T) -> bool { let hash = make_hash(&value); - self.with_get_shard_by_hash(hash, |shard| { let value = value.into_pointer(); shard.raw_entry().from_hash(hash, |entry| entry.into_pointer() == value).is_some() @@ -135,7 +300,6 @@ pub fn make_hash(val: &K) -> u64 { state.finish() } -/* /// Get a shard with a pre-computed hash value. If `get_shard_by_value` is /// ever used in combination with `get_shard_by_hash` on a single `Sharded` /// instance, then `hash` must be computed with `FxHasher`. Otherwise, @@ -143,11 +307,10 @@ pub fn make_hash(val: &K) -> u64 { /// consistently for each `Sharded` instance. #[inline] #[allow(clippy::modulo_one)] -fn get_shard_index_by_hash(hash: u64) -> usize { +pub fn get_shard_index_by_hash(hash: u64) -> usize { let hash_len = mem::size_of::(); // Ignore the top 7 bits as hashbrown uses these and get the next SHARD_BITS highest bits. // hashbrown also uses the lowest bits, so we can't use those let bits = (hash >> (hash_len * 8 - 7 - SHARD_BITS)) as usize; bits % SHARDS } -*/ diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs index 4878c828f4747..d50d1467aa528 100644 --- a/compiler/rustc_data_structures/src/sync.rs +++ b/compiler/rustc_data_structures/src/sync.rs @@ -767,10 +767,6 @@ impl<'a, T> Drop for LockGuard<'a, T> { } } -pub trait SLock: Copy { - type Lock: LockLike; -} - pub trait LockLike { type LockGuard<'a>: DerefMut where @@ -787,13 +783,6 @@ pub trait LockLike { fn lock(&self) -> Self::LockGuard<'_>; } -#[derive(Copy, Clone, Default)] -pub struct SRefCell; - -impl SLock for SRefCell { - type Lock = RefCell; -} - impl LockLike for RefCell { type LockGuard<'a> = RefMut<'a, T> where T: 'a; @@ -824,13 +813,6 @@ impl LockLike for RefCell { } } -#[derive(Copy, Clone, Default)] -pub struct SMutex; - -impl SLock for SMutex { - type Lock = Mutex; -} - impl LockLike for Mutex { type LockGuard<'a> = MutexGuard<'a, T> where T: 'a; diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index 1ab378cbfe998..fb60096ba13f3 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -9,8 +9,16 @@ use crate::query::{ use crate::ty::TyCtxt; use field_offset::FieldOffset; use measureme::StringId; -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sync::{AtomicU64, Lrc, SLock, SMutex, SRefCell, WorkerLocal}; +use rustc_arena::TypedArena; +use rustc_ast as ast; +use rustc_ast::expand::allocator::AllocatorKind; +use rustc_attr as attr; +use rustc_data_structures::fingerprint::Fingerprint; +use rustc_data_structures::fx::{FxHashMap, FxIndexMap, FxIndexSet}; +use rustc_data_structures::sharded::{Shard, Sharded, SingleShard}; +use rustc_data_structures::steal::Steal; +use rustc_data_structures::svh::Svh; +use rustc_data_structures::sync::{AtomicU64, Lrc, WorkerLocal}; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::hir_id::OwnerId; @@ -84,8 +92,8 @@ pub struct QuerySystem<'tcx> { pub dynamic_queries: DynamicQueries<'tcx>, pub single_thread: bool, - pub single_caches: QueryCaches<'tcx, SRefCell>, - pub parallel_caches: QueryCaches<'tcx, SMutex>, + pub single_caches: QueryCaches<'tcx, SingleShard>, + pub parallel_caches: QueryCaches<'tcx, Sharded>, /// This provides access to the incremental compilation on-disk cache for query results. /// Do not access this directly. It is only meant to be used by @@ -377,8 +385,8 @@ macro_rules! define_callbacks { } #[derive(Default)] - pub struct QueryCaches<'tcx, L: SLock> { - $($(#[$attr])* pub $name: query_storage::$name<'tcx, L>,)* + pub struct QueryCaches<'tcx, S: Shard> { + $($(#[$attr])* pub $name: query_storage::$name<'tcx, S>,)* } impl<'tcx> TyCtxtEnsure<'tcx> { diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index e84d0100a5cba..01cff1a3e66eb 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -1310,26 +1310,28 @@ macro_rules! sty_debug_print { }; $(let mut $variant = total;)* - let shards = tcx.interners.type_.lock_shards(); - let types = shards.iter().flat_map(|shard| shard.keys()); - for &InternedInSet(t) in types { - let variant = match t.internee { - ty::Bool | ty::Char | ty::Int(..) | ty::Uint(..) | - ty::Float(..) | ty::Str | ty::Never => continue, - ty::Error(_) => /* unimportant */ continue, - $(ty::$variant(..) => &mut $variant,)* - }; - let lt = t.flags.intersects(ty::TypeFlags::HAS_RE_INFER); - let ty = t.flags.intersects(ty::TypeFlags::HAS_TY_INFER); - let ct = t.flags.intersects(ty::TypeFlags::HAS_CT_INFER); - - variant.total += 1; - total.total += 1; - if lt { total.lt_infer += 1; variant.lt_infer += 1 } - if ty { total.ty_infer += 1; variant.ty_infer += 1 } - if ct { total.ct_infer += 1; variant.ct_infer += 1 } - if lt && ty && ct { total.all_infer += 1; variant.all_infer += 1 } - } + tcx.interners.type_.with_lock_shards(|shard| { + let types = shard.keys(); + for &InternedInSet(t) in types { + let variant = match t.internee { + ty::Bool | ty::Char | ty::Int(..) | ty::Uint(..) | + ty::Float(..) | ty::Str | ty::Never => continue, + ty::Error(_) => /* unimportant */ continue, + $(ty::$variant(..) => &mut $variant,)* + }; + let lt = t.flags.intersects(ty::TypeFlags::HAS_RE_INFER); + let ty = t.flags.intersects(ty::TypeFlags::HAS_TY_INFER); + let ct = t.flags.intersects(ty::TypeFlags::HAS_CT_INFER); + + variant.total += 1; + total.total += 1; + if lt { total.lt_infer += 1; variant.lt_infer += 1 } + if ty { total.ty_infer += 1; variant.ty_infer += 1 } + if ct { total.ct_infer += 1; variant.ct_infer += 1 } + if lt && ty && ct { total.all_infer += 1; variant.all_infer += 1 } + } + }); + writeln!(fmt, "Ty interner total ty lt ct all")?; $(writeln!(fmt, " {:18}: {uses:6} {usespc:4.1}%, \ {ty:4.1}% {lt:5.1}% {ct:4.1}% {all:4.1}%", diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 893149d64560d..e51dc97ffc324 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -17,9 +17,7 @@ extern crate rustc_middle; use crate::plumbing::{encode_all_query_results, try_mark_green}; -use field_offset::offset_of; -use rustc_data_structures::stable_hasher::HashStable; -use rustc_data_structures::sync::{SLock, SMutex, SRefCell}; +use rustc_data_structures::sharded::{Shard, Sharded, SingleShard}; use rustc_middle::arena::Arena; use rustc_middle::dep_graph::DepNodeIndex; use rustc_middle::dep_graph::{self, DepKind, DepKindStruct}; diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index a670084807ef8..4637c56bfa8cb 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -634,10 +634,23 @@ macro_rules! define_queries { + type Cache = query_storage::$name<'tcx, S>; + #[inline(always)] + fn single_query_cache<'a>(self, tcx: QueryCtxt<'tcx>) -> &'a Self::Cache + where 'tcx:'a + { + &tcx.query_system.single_caches.$name + } + #[inline(always)] + fn parallel_query_cache<'a>(self, tcx: QueryCtxt<'tcx>) -> &'a Self::Cache + where 'tcx:'a + { + &tcx.query_system.parallel_caches.$name + } diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index 42fd40c30a4e3..1e6caa0b0897a 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -2,7 +2,7 @@ use parking_lot::Mutex; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_data_structures::profiling::{EventId, QueryInvocationId, SelfProfilerRef}; -use rustc_data_structures::sharded::Sharded; +use rustc_data_structures::sharded::{DynSharded, SHARDS}; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::steal::Steal; use rustc_data_structures::sync::{AtomicU32, AtomicU64, Lock, Lrc, Ordering}; @@ -1083,7 +1083,7 @@ rustc_index::newtype_index! { /// first, and `data` second. pub(super) struct CurrentDepGraph { encoder: Steal>, - new_node_to_index: Sharded, DepNodeIndex>>, + new_node_to_index: DynSharded, DepNodeIndex>>, prev_index_to_index: Lock>>, /// This is used to verify that fingerprints do not change between the creation of a node @@ -1151,7 +1151,11 @@ impl CurrentDepGraph { // doesn't inadvertently increase. static_assert_size!(Option, 4); - let new_node_count_estimate = 102 * prev_graph_node_count / 100 + 200; + let mut new_node_count_estimate = 102 * prev_graph_node_count / 100 + 200; + + if rustc_data_structures::sync::active() { + new_node_count_estimate /= SHARDS; + } let node_intern_event_id = profiler .get_or_alloc_cached_string("incr_comp_intern_dep_graph_node") @@ -1164,7 +1168,7 @@ impl CurrentDepGraph { record_graph, record_stats, )), - new_node_to_index: Sharded::new(|| { + new_node_to_index: DynSharded::new(|| { FxHashMap::with_capacity_and_hasher(new_node_count_estimate, Default::default()) }), prev_index_to_index: Lock::new(IndexVec::from_elem_n(None, prev_graph_node_count)), diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs index adc2808fd1d01..295316a53f9cf 100644 --- a/compiler/rustc_query_system/src/query/caches.rs +++ b/compiler/rustc_query_system/src/query/caches.rs @@ -2,17 +2,18 @@ use crate::dep_graph::DepNodeIndex; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sharded; -use rustc_data_structures::sync::{LockLike, SLock}; +use rustc_data_structures::sharded::{Shard, ShardImpl}; +use rustc_data_structures::sync::LockLike; use rustc_index::{Idx, IndexVec}; use std::fmt::Debug; use std::hash::Hash; use std::marker::PhantomData; -pub trait CacheSelector<'tcx, V, L> { +pub trait CacheSelector<'tcx, V, S> { type Cache where V: Copy, - L: SLock; + S: Shard; } pub trait QueryCache: Sized { @@ -29,23 +30,23 @@ pub trait QueryCache: Sized { pub struct DefaultCacheSelector(PhantomData); -impl<'tcx, K: Eq + Hash, V: 'tcx, L: SLock> CacheSelector<'tcx, V, L> for DefaultCacheSelector { - type Cache = DefaultCache +impl<'tcx, K: Eq + Hash, V: 'tcx, S: Shard> CacheSelector<'tcx, V, S> for DefaultCacheSelector { + type Cache = DefaultCache where V: Copy; } -pub struct DefaultCache { - cache: L::Lock>, +pub struct DefaultCache { + cache: S::Impl>, } -impl Default for DefaultCache { +impl Default for DefaultCache { fn default() -> Self { - DefaultCache { cache: L::Lock::new(FxHashMap::default()) } + DefaultCache { cache: S::Impl::new(|| FxHashMap::default()) } } } -impl QueryCache for DefaultCache +impl QueryCache for DefaultCache where K: Eq + Hash + Copy + Debug, V: Copy, @@ -57,7 +58,7 @@ where fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> { let key_hash = sharded::make_hash(key); - let lock = self.cache.lock(); + let lock = self.cache.get_shard_by_hash(key_hash).lock(); let result = lock.raw_entry().from_key_hashed_nocheck(key_hash, key); @@ -66,7 +67,7 @@ where #[inline] fn complete(&self, key: K, value: V, index: DepNodeIndex) { - let mut lock = self.cache.lock(); + let mut lock = self.cache.get_shard_by_value(&key).lock(); // We may be overwriting another value. This is all right, since the dep-graph // will check that the fingerprint matches. @@ -74,32 +75,34 @@ where } fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) { - let shards = self.cache.lock(); - for (k, v) in shards.iter() { - f(k, &v.0, v.1); + let shards = self.cache.lock_shards(); + for shard in shards.into_iter() { + for (k, v) in shard.iter() { + f(k, &v.0, v.1); + } } } } pub struct SingleCacheSelector; -impl<'tcx, V: 'tcx, L: SLock> CacheSelector<'tcx, V, L> for SingleCacheSelector { - type Cache = SingleCache +impl<'tcx, V: 'tcx, S: Shard> CacheSelector<'tcx, V, S> for SingleCacheSelector { + type Cache = SingleCache where V: Copy; } -pub struct SingleCache { - cache: L::Lock>, +pub struct SingleCache { + cache: > as ShardImpl>>::Lock, } -impl Default for SingleCache { +impl Default for SingleCache { fn default() -> Self { - SingleCache { cache: L::Lock::new(None) } + SingleCache { cache: > as ShardImpl>>::Lock::new(None) } } } -impl QueryCache for SingleCache +impl QueryCache for SingleCache where V: Copy, { @@ -125,23 +128,23 @@ where pub struct VecCacheSelector(PhantomData); -impl<'tcx, K: Idx, V: 'tcx, L: SLock> CacheSelector<'tcx, V, L> for VecCacheSelector { - type Cache = VecCache +impl<'tcx, K: Idx, V: 'tcx, S: Shard> CacheSelector<'tcx, V, S> for VecCacheSelector { + type Cache = VecCache where V: Copy; } -pub struct VecCache { - cache: L::Lock>>, +pub struct VecCache { + cache: S::Impl>>, } -impl Default for VecCache { +impl Default for VecCache { fn default() -> Self { - VecCache { cache: L::Lock::new(IndexVec::default()) } + VecCache { cache: S::Impl::new(|| IndexVec::default()) } } } -impl QueryCache for VecCache +impl QueryCache for VecCache where K: Eq + Idx + Copy + Debug, V: Copy, @@ -151,23 +154,25 @@ where #[inline(always)] fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> { - let lock = self.cache.lock(); + let lock = self.cache.get_shard_by_hash(key.index() as u64).lock(); if let Some(Some(value)) = lock.get(*key) { Some(*value) } else { None } } #[inline] fn complete(&self, key: K, value: V, index: DepNodeIndex) { - let mut lock = self.cache.lock(); + let mut lock = self.cache.get_shard_by_hash(key.index() as u64).lock(); lock.insert(key, (value, index)); } fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) { - let shards = self.cache.lock(); - for (k, v) in shards.iter_enumerated() { - if let Some(v) = v { - f(&k, &v.0, v.1); + let shards = self.cache.lock_shards(); + for shard in shards.iter() { + for (k, v) in shard.iter_enumerated() { + if let Some(v) = v { + f(&k, &v.0, v.1); + } } } } diff --git a/compiler/rustc_query_system/src/query/config.rs b/compiler/rustc_query_system/src/query/config.rs index 5c0f5d5932702..4b696c535228e 100644 --- a/compiler/rustc_query_system/src/query/config.rs +++ b/compiler/rustc_query_system/src/query/config.rs @@ -8,7 +8,7 @@ use crate::query::DepNodeIndex; use crate::query::{QueryContext, QueryInfo, QueryState}; use rustc_data_structures::fingerprint::Fingerprint; -use rustc_data_structures::sync::{SLock, SMutex, SRefCell}; +use rustc_data_structures::sharded::{Shard, Sharded, SingleShard}; use std::fmt::Debug; use std::hash::Hash; @@ -22,7 +22,7 @@ pub trait QueryConfig: Copy { type Key: DepNodeParams + Eq + Hash + Copy + Debug; type Value: Copy; - type Cache: QueryCache; + type Cache: QueryCache; fn single_thread(self, tcx: Qcx) -> bool; @@ -30,11 +30,11 @@ pub trait QueryConfig: Copy { fn look_up(self, tcx: Qcx, key: &Self::Key) -> Option<(Self::Value, DepNodeIndex)>; - fn single_query_cache<'a>(self, tcx: Qcx) -> &'a Self::Cache + fn single_query_cache<'a>(self, tcx: Qcx) -> &'a Self::Cache where Qcx: 'a; - fn parallel_query_cache<'a>(self, tcx: Qcx) -> &'a Self::Cache + fn parallel_query_cache<'a>(self, tcx: Qcx) -> &'a Self::Cache where Qcx: 'a; diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 18999010adf5d..2dfaad95f9763 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -16,9 +16,9 @@ use crate::HandleCycleError; use rustc_data_structures::cold_path; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sharded::Sharded; +use rustc_data_structures::sharded::{DynSharded, Shard, ShardImpl, Sharded, SingleShard}; use rustc_data_structures::stack::ensure_sufficient_stack; -use rustc_data_structures::sync::Lock; +use rustc_data_structures::sync::{Lock, LockLike}; use rustc_errors::{DiagnosticBuilder, ErrorGuaranteed, FatalError}; use rustc_span::{Span, DUMMY_SP}; @@ -34,7 +34,7 @@ use thin_vec::ThinVec; use super::QueryConfig; pub struct QueryState { - active: Sharded>>, + active: DynSharded>>, } /// Indicates the state of a query for a given key in a query map. @@ -53,8 +53,8 @@ where D: DepKind, { pub fn all_inactive(&self) -> bool { - let shards = self.active.lock_shards(); - shards.iter().all(|shard| shard.is_empty()) + let shards = self.active.with_lock_shards(|shard| shard.is_empty()); + shards.into_iter().all(|empty| empty) } pub fn try_collect_active_jobs( @@ -65,15 +65,14 @@ where ) -> Option<()> { // We use try_lock_shards here since we are called from the // deadlock handler, and this shouldn't be locked. - let shards = self.active.try_lock_shards()?; - for shard in shards.iter() { + self.active.with_try_lock_shards(|shard| { for (k, v) in shard.iter() { if let QueryResult::Started(ref job) = *v { let query = make_query(qcx, *k); jobs.insert(job.id, QueryJobInfo { query, job: job.clone() }); } } - } + }); Some(()) } @@ -290,12 +289,38 @@ fn try_execute_query( key: Q::Key, dep_node: Option>, ) -> (Q::Value, Option) +where + Q: QueryConfig, + Qcx: QueryContext, +{ + if query.single_thread(qcx) { + let state = query.query_state(qcx); + let state_lock = state.active.get_borrow_by_value(&key); + try_execute_query_inner::(query, qcx, span, key, dep_node, state_lock) + } else { + let state = query.query_state(qcx); + let state_lock = state.active.get_lock_by_value(&key); + try_execute_query_inner::(query, qcx, span, key, dep_node, state_lock) + } +} + +#[inline(always)] +fn try_execute_query_inner( + query: Q, + qcx: Qcx, + span: Span, + key: Q::Key, + dep_node: Option>, + state_lock: &>> as ShardImpl< + FxHashMap>, + >>::Lock, +) -> (Q::Value, Option) where Q: QueryConfig, Qcx: QueryContext, { let state = query.query_state(qcx); - let mut state_lock = state.active.get_shard_by_value(&key).lock(); + let mut state_lock = state_lock.lock(); let lock = state_lock.deref_mut(); // For the parallel compiler we need to check both the query cache and query state structures From fd51e3539254ca33f2c4b728cfcfc975daf4f38f Mon Sep 17 00:00:00 2001 From: SparrowLii Date: Thu, 18 May 2023 09:01:17 +0800 Subject: [PATCH 10/11] rebase and set default threads to 8 --- compiler/rustc_codegen_ssa/src/base.rs | 2 +- compiler/rustc_data_structures/src/sharded.rs | 10 +- compiler/rustc_data_structures/src/sync.rs | 113 ++++-------------- .../src/sync/worker_local.rs | 6 +- compiler/rustc_interface/src/interface.rs | 2 +- compiler/rustc_interface/src/util.rs | 8 +- compiler/rustc_metadata/src/rmeta/encoder.rs | 2 +- compiler/rustc_middle/src/query/mod.rs | 2 + compiler/rustc_middle/src/query/plumbing.rs | 27 ++--- compiler/rustc_query_impl/src/lib.rs | 97 ++++++++++++--- compiler/rustc_query_impl/src/plumbing.rs | 103 ++++------------ .../rustc_query_system/src/dep_graph/graph.rs | 2 +- .../rustc_query_system/src/query/config.rs | 15 +-- compiler/rustc_query_system/src/query/job.rs | 5 +- compiler/rustc_query_system/src/query/mod.rs | 2 + .../rustc_query_system/src/query/plumbing.rs | 36 +++--- compiler/rustc_session/src/options.rs | 2 +- compiler/rustc_span/src/hygiene.rs | 3 - .../passes/lint/check_code_block_syntax.rs | 11 +- 19 files changed, 189 insertions(+), 259 deletions(-) diff --git a/compiler/rustc_codegen_ssa/src/base.rs b/compiler/rustc_codegen_ssa/src/base.rs index a6fad07f2a2ae..5500541a125f4 100644 --- a/compiler/rustc_codegen_ssa/src/base.rs +++ b/compiler/rustc_codegen_ssa/src/base.rs @@ -686,7 +686,7 @@ pub fn codegen_crate( // This likely is a temporary measure. Once we don't have to support the // non-parallel compiler anymore, we can compile CGUs end-to-end in // parallel and get rid of the complicated scheduling logic. - let mut pre_compiled_cgus = if rustc_data_structures::sync::active() { + let mut pre_compiled_cgus = if rustc_data_structures::sync::is_dyn_thread_safe() { tcx.sess.time("compile_first_CGU_batch", || { // Try to find one CGU to compile per thread. let cgus: Vec<_> = cgu_reuse diff --git a/compiler/rustc_data_structures/src/sharded.rs b/compiler/rustc_data_structures/src/sharded.rs index 381892c057486..dc62934d9368a 100644 --- a/compiler/rustc_data_structures/src/sharded.rs +++ b/compiler/rustc_data_structures/src/sharded.rs @@ -1,5 +1,5 @@ use crate::fx::{FxHashMap, FxHasher}; -use crate::sync::LockLike; +use crate::sync::{DynSync, LockLike}; use parking_lot::{Mutex, MutexGuard}; use std::borrow::Borrow; use std::cell::{RefCell, RefMut}; @@ -131,13 +131,13 @@ pub struct DynSharded { parallel_shard: ShardedImpl, } -// just for speed test -unsafe impl Sync for DynSharded {} +#[cfg(parallel_compiler)] +unsafe impl DynSync for DynSharded {} impl Default for DynSharded { #[inline] fn default() -> Self { - let single_thread = !crate::sync::active(); + let single_thread = !crate::sync::is_dyn_thread_safe(); DynSharded { single_thread, single_shard: RefCell::new(T::default()), @@ -148,7 +148,7 @@ impl Default for DynSharded { impl DynSharded { pub fn new(mut value: impl FnMut() -> T) -> Self { - if !crate::sync::active() { + if !crate::sync::is_dyn_thread_safe() { DynSharded { single_thread: true, single_shard: RefCell::new(value()), diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs index d50d1467aa528..25ff9b89cfa45 100644 --- a/compiler/rustc_data_structures/src/sync.rs +++ b/compiler/rustc_data_structures/src/sync.rs @@ -65,11 +65,10 @@ mod mode { use super::Ordering; use std::sync::atomic::AtomicU8; - const UNINITIALIZED: u8 = 0; const DYN_NOT_THREAD_SAFE: u8 = 1; const DYN_THREAD_SAFE: u8 = 2; - static DYN_THREAD_SAFE_MODE: AtomicU8 = AtomicU8::new(UNINITIALIZED); + static DYN_THREAD_SAFE_MODE: AtomicU8 = AtomicU8::new(DYN_NOT_THREAD_SAFE); // Whether thread safety is enabled (due to running under multiple threads). #[inline] @@ -84,15 +83,9 @@ mod mode { // Only set by the `-Z threads` compile option pub fn set_dyn_thread_safe_mode(mode: bool) { let set: u8 = if mode { DYN_THREAD_SAFE } else { DYN_NOT_THREAD_SAFE }; - let previous = DYN_THREAD_SAFE_MODE.compare_exchange( - UNINITIALIZED, - set, - Ordering::Relaxed, - Ordering::Relaxed, - ); - // Check that the mode was either uninitialized or was already set to the requested mode. - assert!(previous.is_ok() || previous == Err(set)); + // just for speed test + DYN_THREAD_SAFE_MODE.store(set, Ordering::Relaxed); } } @@ -337,24 +330,6 @@ cfg_if! { } } - #[inline] - pub fn join(oper_a: A, oper_b: B) -> (RA, RB) - where - A: FnOnce() -> RA + DynSend, - B: FnOnce() -> RB + DynSend, - { - if mode::active() { - let oper_a = FromDyn::from(oper_a); - let oper_b = FromDyn::from(oper_b); - let (a, b) = rayon::join(move || FromDyn::from(oper_a.into_inner()()), move || FromDyn::from(oper_b.into_inner()())); - (a.into_inner(), b.into_inner()) - } else { - (oper_a(), oper_b()) - } - } - - use std::thread; - #[inline] pub fn join(oper_a: A, oper_b: B) -> (RA, RB) where @@ -385,19 +360,11 @@ cfg_if! { /// the current thread. Use that for the longest running block. #[macro_export] macro_rules! parallel { - (impl $fblock:tt [$($c:tt,)*] [$block:tt $(, $rest:tt)*]) => { (impl $fblock:block [$($c:expr,)*] [$block:expr $(, $rest:expr)*]) => { parallel!(impl $fblock [$block, $($c,)*] [$($rest),*]) }; - (impl $fblock:tt [$($blocks:tt,)*] []) => { (impl $fblock:block [$($blocks:expr,)*] []) => { ::rustc_data_structures::sync::scope(|s| { - - - - - - $(let block = rustc_data_structures::sync::FromDyn::from(|| $blocks); s.spawn(move |_| block.into_inner()());)* (|| $fblock)(); @@ -421,7 +388,6 @@ cfg_if! { } } $( - s.spawn(|_| $blocks); if let Err(p) = ::std::panic::catch_unwind( ::std::panic::AssertUnwindSafe(|| $blocks) ) { @@ -430,19 +396,11 @@ cfg_if! { } } )* - $fblock; - }) if let Some(panic) = panic { ::std::panic::resume_unwind(panic); } } }; - ($fblock:tt, $($blocks:tt),*) => { - // Reverse the order of the later blocks since Rayon executes them in reverse order - // when using a single thread. This ensures the execution order matches that - // of a single threaded rustc - parallel!(impl $fblock [] [$($blocks),*]); - }; } use rayon::iter::{FromParallelIterator, IntoParallelIterator, ParallelIterator}; @@ -453,7 +411,7 @@ cfg_if! { ) { if mode::is_dyn_thread_safe() { let for_each = FromDyn::from(for_each); - let panic: Lock> = Lock::new(None); + let panic: Mutex> = Mutex::new(None); t.into_par_iter().for_each(|i| if let Err(p) = catch_unwind(AssertUnwindSafe(|| for_each(i))) { let mut l = panic.lock(); if l.is_none() { @@ -491,7 +449,7 @@ cfg_if! { map: impl Fn(I) -> R + DynSync + DynSend ) -> C { if mode::is_dyn_thread_safe() { - let panic: Lock> = Lock::new(None); + let panic: Mutex> = Mutex::new(None); let map = FromDyn::from(map); // We catch panics here ensuring that all the loop iterations execute. let r = t.into_par_iter().filter_map(|i| { @@ -531,30 +489,6 @@ cfg_if! { r } } - -pub unsafe trait DynSend {} -pub unsafe trait DynSync {} - -unsafe impl DynSend for T where T: Send {} -unsafe impl DynSync for T where T: Sync {} - -#[derive(Copy, Clone)] -pub struct FromDyn(T); - -impl FromDyn { - #[inline(always)] - pub fn from(val: T) -> Self { - // Check that `sync::active()` is true on creation so we can - // implement `Send` and `Sync` for this structure when `T` - // implements `DynSend` and `DynSync` respectively. - #[cfg(parallel_compiler)] - assert!(mode::active()); - FromDyn(val) - } - - #[inline(always)] - pub fn into_inner(self) -> T { - self.0 } } @@ -607,7 +541,7 @@ impl Lock { #[inline] pub fn new(val: T) -> Self { Lock { - single_thread: !active(), + single_thread: !is_dyn_thread_safe(), data: UnsafeCell::new(val), borrow: Cell::new(false), mutex: RawMutex::INIT, @@ -725,10 +659,6 @@ impl Default for Lock { } } -// Just for speed test -unsafe impl std::marker::Send for Lock {} -unsafe impl std::marker::Sync for Lock {} - pub struct LockGuard<'a, T> { lock: &'a Lock, marker: PhantomData<&'a mut T>, @@ -1037,6 +967,10 @@ pub struct RwLock { data: UnsafeCell, } +// just for speed test +unsafe impl std::marker::Send for RwLock {} +unsafe impl std::marker::Sync for RwLock {} + impl Debug for RwLock { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("Lock").field("data", self.read().deref()).finish() @@ -1046,7 +980,11 @@ impl Debug for RwLock { impl Default for RwLock { fn default() -> Self { RwLock { - raw: RwLockRaw { single_thread: !active(), borrow: Cell::new(0), raw: RawRwLock::INIT }, + raw: RwLockRaw { + single_thread: !is_dyn_thread_safe(), + borrow: Cell::new(0), + raw: RawRwLock::INIT, + }, data: UnsafeCell::new(T::default()), } @@ -1057,7 +995,11 @@ impl RwLock { #[inline(always)] pub fn new(inner: T) -> Self { RwLock { - raw: RwLockRaw { single_thread: !active(), borrow: Cell::new(0), raw: RawRwLock::INIT }, + raw: RwLockRaw { + single_thread: !is_dyn_thread_safe(), + borrow: Cell::new(0), + raw: RawRwLock::INIT, + }, data: UnsafeCell::new(inner), } @@ -1197,10 +1139,6 @@ impl RwLock { } } -// just for speed test -unsafe impl std::marker::Send for RwLock {} -unsafe impl std::marker::Sync for RwLock {} - // FIXME: Probably a bad idea impl Clone for RwLock { #[inline] @@ -1221,7 +1159,7 @@ impl WorkerLocal { /// value this worker local should take for each thread in the thread pool. #[inline] pub fn new T>(mut f: F) -> WorkerLocal { - if !active() { + if !is_dyn_thread_safe() { WorkerLocal { single_thread: true, inner: Some(f(0)), mt_inner: None } } else { WorkerLocal { @@ -1246,9 +1184,6 @@ impl Deref for WorkerLocal { } } -// Just for speed test -unsafe impl std::marker::Sync for WorkerLocal {} - use std::thread; pub use worker_local::Registry; @@ -1261,10 +1196,6 @@ pub struct OneThread { inner: T, } -// just for speed test now -unsafe impl std::marker::Sync for OneThread {} -unsafe impl std::marker::Send for OneThread {} - impl OneThread { #[inline(always)] fn check(&self) { @@ -1273,7 +1204,7 @@ impl OneThread { #[inline(always)] pub fn new(inner: T) -> Self { - OneThread { single_thread: !active(), thread: thread::current().id(), inner } + OneThread { single_thread: !is_dyn_thread_safe(), thread: thread::current().id(), inner } } #[inline(always)] diff --git a/compiler/rustc_data_structures/src/sync/worker_local.rs b/compiler/rustc_data_structures/src/sync/worker_local.rs index c3568c9594338..6de44f01097c2 100644 --- a/compiler/rustc_data_structures/src/sync/worker_local.rs +++ b/compiler/rustc_data_structures/src/sync/worker_local.rs @@ -1,4 +1,4 @@ -use crate::sync::Lock; +use parking_lot::Mutex; use std::cell::Cell; use std::cell::OnceCell; use std::ops::Deref; @@ -34,7 +34,7 @@ impl RegistryId { #[derive(Debug)] struct RegistryData { thread_limit: usize, - threads: Lock, + threads: Mutex, } /// Represents a list of threads which can access worker locals. @@ -64,7 +64,7 @@ thread_local! { impl Registry { /// Creates a registry which can hold up to `thread_limit` threads. pub fn new(thread_limit: usize) -> Self { - Registry(Arc::new(RegistryData { thread_limit, threads: Lock::new(0) })) + Registry(Arc::new(RegistryData { thread_limit, threads: Mutex::new(0) })) } /// Gets the registry associated with the current thread. Panics if there's no such registry. diff --git a/compiler/rustc_interface/src/interface.rs b/compiler/rustc_interface/src/interface.rs index 7f99245bbe686..c8450659cf0e6 100644 --- a/compiler/rustc_interface/src/interface.rs +++ b/compiler/rustc_interface/src/interface.rs @@ -72,7 +72,7 @@ pub fn set_parallel_mode(sopts1: &config::UnstableOptions, sopts2: &config::Code } }; - rustc_data_structures::sync::set(parallel); + rustc_data_structures::sync::set_dyn_thread_safe_mode(parallel); } /// Converts strings provided as `--cfg [cfgspec]` into a `crate_cfg`. diff --git a/compiler/rustc_interface/src/util.rs b/compiler/rustc_interface/src/util.rs index 08e218ea49793..1b2bb1a848c8a 100644 --- a/compiler/rustc_interface/src/util.rs +++ b/compiler/rustc_interface/src/util.rs @@ -6,6 +6,7 @@ use rustc_codegen_ssa::traits::CodegenBackend; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; #[cfg(parallel_compiler)] use rustc_data_structures::sync; +use rustc_data_structures::sync::FromDyn; use rustc_errors::registry::Registry; use rustc_parse::validate_attr; use rustc_session as session; @@ -134,7 +135,7 @@ pub(crate) fn run_in_thread_pool_with_globals R + Send, R: Send>( f: F, ) -> R { #[cfg(parallel_compiler)] - if rustc_data_structures::sync::active() { + if rustc_data_structures::sync::is_dyn_thread_safe() { return run_in_threads_pool_with_globals(edition, _threads, f); } // The "thread pool" is a single spawned thread in the non-parallel @@ -203,6 +204,7 @@ pub(crate) fn run_in_threads_pool_with_globals R + Send, R: Send> // `Send` in the parallel compiler. rustc_span::create_session_globals_then(edition, || { rustc_span::with_session_globals(|session_globals| { + let session_globals = FromDyn::from(session_globals); builder .build_scoped( // Initialize each new worker thread when created. @@ -210,7 +212,9 @@ pub(crate) fn run_in_threads_pool_with_globals R + Send, R: Send> // Register the thread for use with the `WorkerLocal` type. registry.register(); - rustc_span::set_session_globals_then(session_globals, || thread.run()) + rustc_span::set_session_globals_then(session_globals.into_inner(), || { + thread.run() + }) }, // Run `f` on the first thread in the thread pool. move |pool: &rayon::ThreadPool| pool.install(f), diff --git a/compiler/rustc_metadata/src/rmeta/encoder.rs b/compiler/rustc_metadata/src/rmeta/encoder.rs index ed3a9637eab43..f663f24dfb76e 100644 --- a/compiler/rustc_metadata/src/rmeta/encoder.rs +++ b/compiler/rustc_metadata/src/rmeta/encoder.rs @@ -2233,7 +2233,7 @@ pub fn encode_metadata(tcx: TyCtxt<'_>, path: &Path) { join( || encode_metadata_impl(tcx, path), || { - if !rustc_data_structures::sync::active() { + if !rustc_data_structures::sync::is_dyn_thread_safe() { return; } // Prefetch some queries used by metadata encoding. diff --git a/compiler/rustc_middle/src/query/mod.rs b/compiler/rustc_middle/src/query/mod.rs index 21c69662b9ead..e14aec9772ba2 100644 --- a/compiler/rustc_middle/src/query/mod.rs +++ b/compiler/rustc_middle/src/query/mod.rs @@ -54,6 +54,7 @@ use rustc_ast::expand::allocator::AllocatorKind; use rustc_attr as attr; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::{FxHashMap, FxIndexMap, FxIndexSet}; +use rustc_data_structures::sharded::{Shard, Sharded, SingleShard}; use rustc_data_structures::steal::Steal; use rustc_data_structures::svh::Svh; use rustc_data_structures::sync::Lrc; @@ -80,6 +81,7 @@ use rustc_span::symbol::Symbol; use rustc_span::{Span, DUMMY_SP}; use rustc_target::abi; use rustc_target::spec::PanicStrategy; +use std::intrinsics::likely; use std::mem; use std::ops::Deref; use std::path::PathBuf; diff --git a/compiler/rustc_middle/src/query/plumbing.rs b/compiler/rustc_middle/src/query/plumbing.rs index fb60096ba13f3..d7618b971e737 100644 --- a/compiler/rustc_middle/src/query/plumbing.rs +++ b/compiler/rustc_middle/src/query/plumbing.rs @@ -9,16 +9,9 @@ use crate::query::{ use crate::ty::TyCtxt; use field_offset::FieldOffset; use measureme::StringId; -use rustc_arena::TypedArena; -use rustc_ast as ast; -use rustc_ast::expand::allocator::AllocatorKind; -use rustc_attr as attr; -use rustc_data_structures::fingerprint::Fingerprint; -use rustc_data_structures::fx::{FxHashMap, FxIndexMap, FxIndexSet}; -use rustc_data_structures::sharded::{Shard, Sharded, SingleShard}; -use rustc_data_structures::steal::Steal; -use rustc_data_structures::svh::Svh; -use rustc_data_structures::sync::{AtomicU64, Lrc, WorkerLocal}; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sharded::{Sharded, SingleShard}; +use rustc_data_structures::sync::{AtomicU64, DynSync}; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::hir_id::OwnerId; @@ -28,7 +21,6 @@ pub(crate) use rustc_query_system::query::QueryJobId; use rustc_query_system::query::*; use rustc_query_system::HandleCycleError; use rustc_span::{Span, DUMMY_SP}; -use std::intrinsics::likely; use std::ops::Deref; pub struct QueryKeyStringCache { @@ -49,13 +41,14 @@ pub struct QueryStruct<'tcx> { Option, &mut CacheEncoder<'_, 'tcx>, &mut EncodedDepNodeIndex)>, } -pub struct DynamicQuery<'tcx, C: QueryCache> { +pub struct DynamicQuery<'tcx, C: QueryCache, C2: QueryCache> { pub name: &'static str, pub eval_always: bool, pub dep_kind: rustc_middle::dep_graph::DepKind, pub handle_cycle_error: HandleCycleError, pub query_state: FieldOffset, QueryState>, - pub query_cache: FieldOffset, C>, + pub single_query_cache: FieldOffset, C>, + pub parallel_query_cache: FieldOffset, C2>, pub cache_on_disk: fn(tcx: TyCtxt<'tcx>, key: &C::Key) -> bool, pub execute_query: fn(tcx: TyCtxt<'tcx>, k: C::Key) -> C::Value, pub compute: fn(tcx: TyCtxt<'tcx>, key: C::Key) -> C::Value, @@ -86,6 +79,7 @@ pub struct QuerySystemFns<'tcx> { ), pub try_mark_green: fn(tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool, } + pub struct QuerySystem<'tcx> { pub states: QueryStates<'tcx>, pub arenas: QueryArenas<'tcx>, @@ -106,6 +100,9 @@ pub struct QuerySystem<'tcx> { pub jobs: AtomicU64, } +#[cfg(parallel_compiler)] +unsafe impl<'tcx> DynSync for QuerySystem<'tcx> {} + #[derive(Copy, Clone)] pub struct TyCtxtAt<'tcx> { pub tcx: TyCtxt<'tcx>, @@ -332,7 +329,7 @@ macro_rules! define_callbacks { use super::*; $( - pub type $name<'tcx, L> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>, L>>::Cache; + pub type $name<'tcx, S> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, Erase<$V>, S>>::Cache; )* } @@ -474,7 +471,7 @@ macro_rules! define_callbacks { pub struct DynamicQueries<'tcx> { $( - pub $name: DynamicQuery<'tcx, query_storage::$name<'tcx>>, + pub $name: DynamicQuery<'tcx, query_storage::$name<'tcx, SingleShard>, query_storage::$name<'tcx, Sharded>>, )* } diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index e51dc97ffc324..b1d9d57429944 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -17,9 +17,11 @@ extern crate rustc_middle; use crate::plumbing::{encode_all_query_results, try_mark_green}; +use field_offset::offset_of; use rustc_data_structures::sharded::{Shard, Sharded, SingleShard}; +use rustc_data_structures::stable_hasher::HashStable; +use rustc_data_structures::sync::AtomicU64; use rustc_middle::arena::Arena; -use rustc_middle::dep_graph::DepNodeIndex; use rustc_middle::dep_graph::{self, DepKind, DepKindStruct}; use rustc_middle::query::erase::{erase, restore, Erase}; use rustc_middle::query::on_disk_cache::OnDiskCache; @@ -33,13 +35,12 @@ use rustc_middle::ty::TyCtxt; use rustc_query_system::dep_graph::{DepNodeIndex, SerializedDepNodeIndex}; use rustc_query_system::ich::StableHashingContext; use rustc_query_system::query::{ - get_query_incr, get_query_non_incr, HashResult, QueryCache, QueryConfig, QueryInfo, QueryMap, - QueryMode, QueryState, + get_query_incr, get_query_non_incr, HashResult, QueryCache, QueryConfig, QueryContext, + QueryInfo, QueryMap, QueryMode, QueryState, }; use rustc_query_system::HandleCycleError; use rustc_query_system::Value; use rustc_span::Span; -use std::intrinsics::likely; #[macro_use] mod plumbing; @@ -51,33 +52,52 @@ pub use self::profiling_support::alloc_self_profile_query_strings; struct DynamicConfig< 'tcx, C: QueryCache, + C2: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool, > { - dynamic: &'tcx DynamicQuery<'tcx, C>, + dynamic: &'tcx DynamicQuery<'tcx, C, C2>, } -impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Copy - for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE> +impl< + 'tcx, + C: QueryCache, + C2: QueryCache, + const ANON: bool, + const DEPTH_LIMIT: bool, + const FEEDABLE: bool, +> Copy for DynamicConfig<'tcx, C, C2, ANON, DEPTH_LIMIT, FEEDABLE> { } -impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> Clone - for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE> +impl< + 'tcx, + C: QueryCache, + C2: QueryCache, + const ANON: bool, + const DEPTH_LIMIT: bool, + const FEEDABLE: bool, +> Clone for DynamicConfig<'tcx, C, C2, ANON, DEPTH_LIMIT, FEEDABLE> { fn clone(&self) -> Self { DynamicConfig { dynamic: self.dynamic } } } -impl<'tcx, C: QueryCache, const ANON: bool, const DEPTH_LIMIT: bool, const FEEDABLE: bool> - QueryConfig> for DynamicConfig<'tcx, C, ANON, DEPTH_LIMIT, FEEDABLE> +impl< + 'tcx, + C: QueryCache, + C2: QueryCache, + const ANON: bool, + const DEPTH_LIMIT: bool, + const FEEDABLE: bool, +> QueryConfig> for DynamicConfig<'tcx, C, C2, ANON, DEPTH_LIMIT, FEEDABLE> where for<'a> C::Key: HashStable>, { type Key = C::Key; type Value = C::Value; - type Cache = C; + type Cache = C; #[inline(always)] fn name(self) -> &'static str { @@ -98,11 +118,49 @@ where } #[inline(always)] - fn query_cache<'a>(self, qcx: QueryCtxt<'tcx>) -> &'a Self::Cache - where - 'tcx: 'a, - { - self.dynamic.query_cache.apply(&qcx.tcx.query_system.caches) + fn look_up(self, qcx: QueryCtxt<'tcx>, key: &Self::Key) -> Option<(Self::Value, DepNodeIndex)> { + if qcx.single_thread() { + self.dynamic.single_query_cache.apply(&qcx.tcx.query_system.single_caches).lookup(key) + } else { + self.dynamic + .parallel_query_cache + .apply(&qcx.tcx.query_system.parallel_caches) + .lookup(key) + } + } + + #[inline(always)] + fn cache_iter( + self, + qcx: QueryCtxt<'tcx>, + f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex), + ) { + if qcx.single_thread() { + self.dynamic.single_query_cache.apply(&qcx.tcx.query_system.single_caches).iter(f) + } else { + self.dynamic.parallel_query_cache.apply(&qcx.tcx.query_system.parallel_caches).iter(f) + } + } + + #[inline(always)] + fn complete( + self, + qcx: QueryCtxt<'tcx>, + key: Self::Key, + value: Self::Value, + index: DepNodeIndex, + ) { + if qcx.single_thread() { + self.dynamic + .single_query_cache + .apply(&qcx.tcx.query_system.single_caches) + .complete(key, value, index) + } else { + self.dynamic + .parallel_query_cache + .apply(&qcx.tcx.query_system.parallel_caches) + .complete(key, value, index) + } } #[inline(always)] @@ -209,8 +267,11 @@ pub fn query_system<'tcx>( QuerySystem { states: Default::default(), arenas: Default::default(), - caches: Default::default(), dynamic_queries: dynamic_queries(), + + single_thread: !rustc_data_structures::sync::is_dyn_thread_safe(), + single_caches: Default::default(), + parallel_caches: Default::default(), on_disk_cache, fns: QuerySystemFns { engine: engine(incremental), diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index 4637c56bfa8cb..2604212b4fd98 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -20,15 +20,13 @@ use rustc_middle::ty::{self, TyCtxt}; use rustc_query_system::dep_graph::{DepNodeParams, HasDepContext}; use rustc_query_system::ich::StableHashingContext; use rustc_query_system::query::{ - force_query, QueryCache, QueryConfig, QueryContext, QueryJobId, QueryMap, QuerySideEffects, - QueryStackFrame, + force_query, QueryConfig, QueryContext, QueryJobId, QueryMap, QuerySideEffects, QueryStackFrame, }; use rustc_query_system::{LayoutOfDepth, QueryOverflow}; use rustc_serialize::Decodable; use rustc_serialize::Encodable; use rustc_session::Limit; use rustc_span::def_id::LOCAL_CRATE; -use std::intrinsics::likely; use std::num::NonZeroU64; use thin_vec::ThinVec; @@ -64,6 +62,11 @@ impl<'tcx> HasDepContext for QueryCtxt<'tcx> { } impl QueryContext for QueryCtxt<'_> { + #[inline] + fn single_thread(self) -> bool { + self.query_system.single_thread + } + #[inline] fn next_job_id(self) -> QueryJobId { QueryJobId( @@ -355,35 +358,18 @@ pub(crate) fn encode_query_results<'a, 'tcx, Q>( qcx.profiler().verbose_generic_activity_with_arg("encode_query_results_for", query.name()); assert!(query.query_state(qcx).all_inactive()); - if likely(query.single_thread(qcx)) { - let cache = query.single_query_cache(qcx); - cache.iter(&mut |key, value, dep_node| { - if query.cache_on_disk(qcx.tcx, &key) { - let dep_node = SerializedDepNodeIndex::new(dep_node.index()); - - // Record position of the cache entry. - query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position()))); - - // Encode the type check tables with the `SerializedDepNodeIndex` - // as tag. - encoder.encode_tagged(dep_node, &Q::restore(*value)); - } - }); - } else { - let cache = query.parallel_query_cache(qcx); - cache.iter(&mut |key, value, dep_node| { - if query.cache_on_disk(qcx.tcx, &key) { - let dep_node = SerializedDepNodeIndex::new(dep_node.index()); + query.cache_iter(qcx, &mut |key, value, dep_node: DepNodeIndex| { + if query.cache_on_disk(qcx.tcx, &key) { + let dep_node = SerializedDepNodeIndex::new(dep_node.index()); - // Record position of the cache entry. - query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position()))); + // Record position of the cache entry. + query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position()))); - // Encode the type check tables with the `SerializedDepNodeIndex` - // as tag. - encoder.encode_tagged(dep_node, &Q::restore(*value)); - } - }); - } + // Encode the type check tables with the `SerializedDepNodeIndex` + // as tag. + encoder.encode_tagged(dep_node, &Q::restore(*value)); + } + }); } fn try_load_from_on_disk_cache<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode) @@ -585,14 +571,15 @@ macro_rules! define_queries { use super::*; $( - pub(super) fn $name<'tcx>() -> DynamicQuery<'tcx, query_storage::$name<'tcx>> { + pub(super) fn $name<'tcx>() -> DynamicQuery<'tcx, query_storage::$name<'tcx, SingleShard>, query_storage::$name<'tcx, Sharded>> { DynamicQuery { name: stringify!($name), eval_always: is_eval_always!([$($modifiers)*]), dep_kind: dep_graph::DepKind::$name, handle_cycle_error: handle_cycle_error!([$($modifiers)*]), query_state: offset_of!(QueryStates<'tcx> => $name), - query_cache: offset_of!(QueryCaches<'tcx> => $name), + single_query_cache: offset_of!(QueryCaches<'tcx, SingleShard> => $name), + parallel_query_cache: offset_of!(QueryCaches<'tcx, Sharded> => $name), cache_on_disk: |tcx, key| ::rustc_middle::query::cached::$name(tcx, key), execute_query: |tcx, key| erase(tcx.$name(key)), compute: |tcx, key| query_provided_to_value::$name( @@ -630,55 +617,6 @@ macro_rules! define_queries { }, hash_result: hash_result!([$($modifiers)*][query_values::$name<'tcx>]), format_value: |value| format!("{:?}", restore::>(*value)), - - - - - type Cache = query_storage::$name<'tcx, S>; - - - - #[inline(always)] - fn single_query_cache<'a>(self, tcx: QueryCtxt<'tcx>) -> &'a Self::Cache - where 'tcx:'a - { - &tcx.query_system.single_caches.$name - } - - #[inline(always)] - fn parallel_query_cache<'a>(self, tcx: QueryCtxt<'tcx>) -> &'a Self::Cache - where 'tcx:'a - { - &tcx.query_system.parallel_caches.$name - } - - - - - - - - - - - - - - - - - - - - - - - - - - - - } } )* @@ -688,7 +626,8 @@ macro_rules! define_queries { type RestoredValue = query_values::$name<'tcx>; type Config = DynamicConfig< 'tcx, - query_storage::$name<'tcx>, + query_storage::$name<'tcx, SingleShard>, + query_storage::$name<'tcx, Sharded>, { is_anon!([$($modifiers)*]) }, { depth_limit!([$($modifiers)*]) }, { feedable!([$($modifiers)*]) }, diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index 1e6caa0b0897a..c5a144eb42473 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -1153,7 +1153,7 @@ impl CurrentDepGraph { let mut new_node_count_estimate = 102 * prev_graph_node_count / 100 + 200; - if rustc_data_structures::sync::active() { + if rustc_data_structures::sync::is_dyn_thread_safe() { new_node_count_estimate /= SHARDS; } diff --git a/compiler/rustc_query_system/src/query/config.rs b/compiler/rustc_query_system/src/query/config.rs index 4b696c535228e..72bb673a91021 100644 --- a/compiler/rustc_query_system/src/query/config.rs +++ b/compiler/rustc_query_system/src/query/config.rs @@ -4,11 +4,10 @@ use crate::dep_graph::{DepNode, DepNodeIndex, DepNodeParams, SerializedDepNodeIn use crate::error::HandleCycleError; use crate::ich::StableHashingContext; use crate::query::caches::QueryCache; -use crate::query::DepNodeIndex; use crate::query::{QueryContext, QueryInfo, QueryState}; use rustc_data_structures::fingerprint::Fingerprint; -use rustc_data_structures::sharded::{Shard, Sharded, SingleShard}; +use rustc_data_structures::sharded::Shard; use std::fmt::Debug; use std::hash::Hash; @@ -24,19 +23,13 @@ pub trait QueryConfig: Copy { type Cache: QueryCache; - fn single_thread(self, tcx: Qcx) -> bool; - fn format_value(self) -> fn(&Self::Value) -> String; - fn look_up(self, tcx: Qcx, key: &Self::Key) -> Option<(Self::Value, DepNodeIndex)>; + fn look_up(self, qcx: Qcx, key: &Self::Key) -> Option<(Self::Value, DepNodeIndex)>; - fn single_query_cache<'a>(self, tcx: Qcx) -> &'a Self::Cache - where - Qcx: 'a; + fn cache_iter(self, qcx: Qcx, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)); - fn parallel_query_cache<'a>(self, tcx: Qcx) -> &'a Self::Cache - where - Qcx: 'a; + fn complete(self, qcx: Qcx, key: Self::Key, value: Self::Value, index: DepNodeIndex); // Don't use this method to access query results, instead use the methods on TyCtxt fn query_state<'a>(self, tcx: Qcx) -> &'a QueryState diff --git a/compiler/rustc_query_system/src/query/job.rs b/compiler/rustc_query_system/src/query/job.rs index 0dc103c015bb7..d731ef8121a44 100644 --- a/compiler/rustc_query_system/src/query/job.rs +++ b/compiler/rustc_query_system/src/query/job.rs @@ -20,7 +20,6 @@ use { parking_lot::{Condvar, Mutex}, rayon_core, rustc_data_structures::fx::FxHashSet, - rustc_data_structures::sync::Lock, rustc_data_structures::sync::Lrc, rustc_data_structures::{jobserver, OnDrop}, rustc_span::DUMMY_SP, @@ -189,7 +188,7 @@ struct QueryWaiter { query: Option, condvar: Condvar, span: Span, - cycle: Lock>>, + cycle: Mutex>>, } #[cfg(parallel_compiler)] @@ -227,7 +226,7 @@ impl QueryLatch { span: Span, ) -> Result<(), CycleError> { let waiter = - Lrc::new(QueryWaiter { query, span, cycle: Lock::new(None), condvar: Condvar::new() }); + Lrc::new(QueryWaiter { query, span, cycle: Mutex::new(None), condvar: Condvar::new() }); self.wait_on_inner(&waiter); // FIXME: Get rid of this lock. We have ownership of the QueryWaiter // although another thread may still have a Lrc reference so we cannot diff --git a/compiler/rustc_query_system/src/query/mod.rs b/compiler/rustc_query_system/src/query/mod.rs index f7619d75be768..cc78ed871152d 100644 --- a/compiler/rustc_query_system/src/query/mod.rs +++ b/compiler/rustc_query_system/src/query/mod.rs @@ -101,6 +101,8 @@ impl QuerySideEffects { } pub trait QueryContext: HasDepContext { + fn single_thread(self) -> bool; + fn next_job_id(self) -> QueryJobId; /// Get the query information from the TLS context. diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 2dfaad95f9763..8b7ced1298c5d 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -26,7 +26,6 @@ use std::cell::Cell; use std::collections::hash_map::Entry; use std::fmt::Debug; use std::hash::Hash; -use std::intrinsics::likely; use std::mem; use std::ops::DerefMut; use thin_vec::ThinVec; @@ -145,9 +144,14 @@ where { /// Completes the query by updating the query cache with the `result`, /// signals the waiter and forgets the JobOwner, so it won't poison the query - fn complete(self, cache: &C, result: C::Value, dep_node_index: DepNodeIndex) - where - C: QueryCache, + fn complete( + self, + qcx: Qcx, + query: C, + result: C::Value, + dep_node_index: DepNodeIndex, + ) where + C: QueryConfig, { let key = self.key; let state = self.state; @@ -157,7 +161,7 @@ where // Mark as complete before we remove the job from the active state // so no other thread can re-execute this query. - cache.complete(key, result, dep_node_index); + query.complete(qcx, key, result, dep_node_index); let job = { state.active.with_get_shard_by_value(&key, |lock| match lock.remove(&key).unwrap() { @@ -293,19 +297,23 @@ where Q: QueryConfig, Qcx: QueryContext, { - if query.single_thread(qcx) { + if qcx.single_thread() { let state = query.query_state(qcx); let state_lock = state.active.get_borrow_by_value(&key); - try_execute_query_inner::(query, qcx, span, key, dep_node, state_lock) + try_execute_query_inner::( + query, qcx, span, key, dep_node, state_lock, + ) } else { let state = query.query_state(qcx); let state_lock = state.active.get_lock_by_value(&key); - try_execute_query_inner::(query, qcx, span, key, dep_node, state_lock) + try_execute_query_inner::( + query, qcx, span, key, dep_node, state_lock, + ) } } #[inline(always)] -fn try_execute_query_inner( +fn try_execute_query_inner( query: Q, qcx: Qcx, span: Span, @@ -329,7 +337,7 @@ where // re-executing the query since `try_start` only checks that the query is not currently // executing, but another thread may have already completed the query and stores it result // in the query cache. - if cfg!(parallel_compiler) && rustc_data_structures::sync::active() { + if cfg!(parallel_compiler) && rustc_data_structures::sync::is_dyn_thread_safe() { if let Some((value, index)) = query.look_up(qcx, &key) { qcx.dep_context().profiler().query_cache_hit(index.into()); return (value, Some(index)); @@ -364,7 +372,7 @@ where } #[cfg(parallel_compiler)] QueryResult::Started(job) => { - if std::intrinsics::likely(!rustc_data_structures::sync::active()) { + if std::intrinsics::likely(!rustc_data_structures::sync::is_dyn_thread_safe()) { let id = job.id; drop(state_lock); @@ -447,11 +455,7 @@ where } } - if likely(query.single_thread(qcx)) { - job_owner.complete(query.single_query_cache(qcx), result, dep_node_index) - } else { - job_owner.complete(query.parallel_query_cache(qcx), result, dep_node_index) - }; + job_owner.complete(qcx, query, result, dep_node_index); (result, Some(dep_node_index)) } diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 17d9a95a10da2..060da8ada795e 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1748,7 +1748,7 @@ options! { /// in the future. Note that -Zthreads=0 is the way to get /// the num_cpus behavior. #[rustc_lint_opt_deny_field_access("use `Session::threads` instead of this field")] - threads: usize = (2, parse_threads, [UNTRACKED], + threads: usize = (8, parse_threads, [UNTRACKED], "use a thread pool with N threads"), time_llvm_passes: bool = (false, parse_bool, [UNTRACKED], "measure time of each LLVM pass (default: no)"), diff --git a/compiler/rustc_span/src/hygiene.rs b/compiler/rustc_span/src/hygiene.rs index c669b64dd2c81..fbc2d64534824 100644 --- a/compiler/rustc_span/src/hygiene.rs +++ b/compiler/rustc_span/src/hygiene.rs @@ -1278,11 +1278,8 @@ pub fn register_expn_id( let expn_id = ExpnId { krate, local_id }; HygieneData::with(|hygiene_data| { let _old_data = hygiene_data.foreign_expn_data.insert(expn_id, data); - debug_assert!(_old_data.is_none()); let _old_hash = hygiene_data.foreign_expn_hashes.insert(expn_id, hash); - debug_assert!(_old_hash.is_none()); let _old_id = hygiene_data.expn_hash_to_expn_id.insert(hash, expn_id); - debug_assert!(_old_id.is_none()); }); expn_id } diff --git a/src/librustdoc/passes/lint/check_code_block_syntax.rs b/src/librustdoc/passes/lint/check_code_block_syntax.rs index 8f873dbe50131..b74d3fe749d5e 100644 --- a/src/librustdoc/passes/lint/check_code_block_syntax.rs +++ b/src/librustdoc/passes/lint/check_code_block_syntax.rs @@ -1,5 +1,5 @@ //! Validates syntax inside Rust code blocks (\`\`\`rust). -use rustc_data_structures::sync::{Lock, Lrc}; +use rustc_data_structures::sync::Lrc; use rustc_errors::{ emitter::Emitter, translation::{to_fluent_args, Translate}, @@ -10,6 +10,7 @@ use rustc_session::parse::ParseSess; use rustc_span::hygiene::{AstPass, ExpnData, ExpnKind, LocalExpnId}; use rustc_span::source_map::{FilePathMapping, SourceMap}; use rustc_span::{FileName, InnerSpan, DUMMY_SP}; +use std::sync::Mutex; use crate::clean; use crate::core::DocContext; @@ -32,7 +33,7 @@ fn check_rust_syntax( dox: &str, code_block: RustCodeBlock, ) { - let buffer = Lrc::new(Lock::new(Buffer::default())); + let buffer = Lrc::new(Mutex::new(Buffer::default())); let fallback_bundle = rustc_errors::fallback_fluent_bundle( rustc_driver::DEFAULT_LOCALE_RESOURCES.to_vec(), false, @@ -60,7 +61,7 @@ fn check_rust_syntax( .is_empty() }) .unwrap_or(false); - let buffer = buffer.borrow(); + let buffer = buffer.lock().unwrap(); if !buffer.has_errors && !is_empty { // No errors in a non-empty program. @@ -138,7 +139,7 @@ struct Buffer { } struct BufferEmitter { - buffer: Lrc>, + buffer: Lrc>, fallback_bundle: LazyFallbackBundle, } @@ -154,7 +155,7 @@ impl Translate for BufferEmitter { impl Emitter for BufferEmitter { fn emit_diagnostic(&mut self, diag: &Diagnostic) { - let mut buffer = self.buffer.borrow_mut(); + let mut buffer = self.buffer.lock().unwrap(); let fluent_args = to_fluent_args(diag.args()); let translated_main_message = self From 0ed54d5cc0d3506686dea23bd55311987f65d180 Mon Sep 17 00:00:00 2001 From: SparrowLii Date: Thu, 18 May 2023 17:19:42 +0800 Subject: [PATCH 11/11] test for threads = 1 again --- compiler/rustc_data_structures/Cargo.toml | 4 +- compiler/rustc_data_structures/src/lib.rs | 1 - compiler/rustc_data_structures/src/sync.rs | 44 +----- compiler/rustc_query_system/src/cache.rs | 6 +- .../rustc_query_system/src/dep_graph/graph.rs | 134 +++++++++--------- .../src/dep_graph/serialized.rs | 6 +- .../src/ich/impls_syntax.rs | 2 +- compiler/rustc_session/src/options.rs | 2 +- compiler/rustc_session/src/parse.rs | 22 ++- 9 files changed, 85 insertions(+), 136 deletions(-) diff --git a/compiler/rustc_data_structures/Cargo.toml b/compiler/rustc_data_structures/Cargo.toml index b92d57ea4ad2d..afd354112bbf0 100644 --- a/compiler/rustc_data_structures/Cargo.toml +++ b/compiler/rustc_data_structures/Cargo.toml @@ -14,7 +14,7 @@ indexmap = { version = "1.9.3" } jobserver_crate = { version = "0.1.13", package = "jobserver" } libc = "0.2" measureme = "10.0.0" -rustc-rayon-core = { version = "0.5.0" } +rustc-rayon-core = { version = "0.5.0", optional = true } rustc-rayon = { version = "0.5.0", optional = true } rustc_arena = { path = "../rustc_arena" } rustc_graphviz = { path = "../rustc_graphviz" } @@ -51,4 +51,4 @@ features = [ memmap2 = "0.2.1" [features] -rustc_use_parallel_compiler = ["indexmap/rustc-rayon", "rustc-rayon"] \ No newline at end of file +rustc_use_parallel_compiler = ["indexmap/rustc-rayon", "rustc-rayon", "rustc-rayon-core"] \ No newline at end of file diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs index fdd09d59a76c2..5b9b0e106d254 100644 --- a/compiler/rustc_data_structures/src/lib.rs +++ b/compiler/rustc_data_structures/src/lib.rs @@ -33,7 +33,6 @@ #![feature(strict_provenance)] #![feature(ptr_alignment_type)] #![feature(macro_metavar_expr)] -#![feature(mutex_unpoison)] #![allow(rustc::default_hash_types)] #![allow(rustc::potential_query_instability)] #![deny(rustc::untranslatable_diagnostic)] diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs index 25ff9b89cfa45..f0e41873830dc 100644 --- a/compiler/rustc_data_structures/src/sync.rs +++ b/compiler/rustc_data_structures/src/sync.rs @@ -493,7 +493,7 @@ cfg_if! { } #[derive(Default, Debug)] -#[repr(align(64))] +#[cfg_attr(parallel_compiler, repr(align(64)))] pub struct CacheAligned(pub T); pub trait HashMapExt { @@ -593,50 +593,10 @@ impl Lock { LockGuard { lock: &self, marker: PhantomData } } - #[inline] - pub(crate) fn with_mt_lock R, R>(&self, f: F) -> R { - unsafe { - self.mutex.lock(); - let r = f(&mut *self.data.get()); - self.mutex.unlock(); - r - } - } - #[inline(always)] #[track_caller] pub fn with_lock R, R>(&self, f: F) -> R { - if likely(self.single_thread) { - assert!(!self.borrow.replace(true)); - let r = unsafe { f(&mut *self.data.get()) }; - self.borrow.set(false); - r - } else { - self.with_mt_lock(f) - } - } - - #[inline] - fn with_mt_borrow R, R>(&self, f: F) -> R { - unsafe { - self.mutex.lock(); - let r = f(&*self.data.get()); - self.mutex.unlock(); - r - } - } - - #[inline(always)] - #[track_caller] - pub fn with_borrow R, R>(&self, f: F) -> R { - if likely(self.single_thread) { - assert!(!self.borrow.replace(true)); - let r = unsafe { f(&*self.data.get()) }; - self.borrow.set(false); - r - } else { - self.with_mt_borrow(f) - } + f(&mut *self.lock()) } #[inline(always)] diff --git a/compiler/rustc_query_system/src/cache.rs b/compiler/rustc_query_system/src/cache.rs index e06579eb0ba63..6e862db0b2547 100644 --- a/compiler/rustc_query_system/src/cache.rs +++ b/compiler/rustc_query_system/src/cache.rs @@ -26,17 +26,17 @@ impl Default for Cache { impl Cache { /// Actually frees the underlying memory in contrast to what stdlib containers do on `clear` pub fn clear(&self) { - self.hashmap.with_lock(|map| *map = Default::default()); + *self.hashmap.borrow_mut() = Default::default(); } } impl Cache { pub fn get(&self, key: &Key, tcx: Tcx) -> Option { - self.hashmap.with_borrow(|map| map.get(key).map(|node| node.get(tcx))) + Some(self.hashmap.borrow().get(key)?.get(tcx)) } pub fn insert(&self, key: Key, dep_node: DepNodeIndex, value: Value) { - self.hashmap.with_lock(|map| map.insert(key, WithDepNode::new(dep_node, value))); + self.hashmap.borrow_mut().insert(key, WithDepNode::new(dep_node, value)); } } diff --git a/compiler/rustc_query_system/src/dep_graph/graph.rs b/compiler/rustc_query_system/src/dep_graph/graph.rs index c5a144eb42473..96f8b08af6c74 100644 --- a/compiler/rustc_query_system/src/dep_graph/graph.rs +++ b/compiler/rustc_query_system/src/dep_graph/graph.rs @@ -466,42 +466,8 @@ impl DepGraph { pub fn read_index(&self, dep_node_index: DepNodeIndex) { if let Some(ref data) = self.data { K::read_deps(|task_deps| { - match task_deps { - TaskDepsRef::Allow(deps) => deps.with_lock(|task_deps| { - // As long as we only have a low number of reads we can avoid doing a hash - // insert and potentially allocating/reallocating the hashmap - let new_read = if task_deps.reads.len() < TASK_DEPS_READS_CAP { - task_deps.reads.iter().all(|other| *other != dep_node_index) - } else { - task_deps.read_set.insert(dep_node_index) - }; - if new_read { - task_deps.reads.push(dep_node_index); - if task_deps.reads.len() == TASK_DEPS_READS_CAP { - // Fill `read_set` with what we have so far so we can use the hashset - // next time - task_deps.read_set.extend(task_deps.reads.iter().copied()); - } - - #[cfg(debug_assertions)] - { - if let Some(target) = task_deps.node { - if let Some(ref forbidden_edge) = data.current.forbidden_edge { - let src = - forbidden_edge.index_to_node.lock()[&dep_node_index]; - if forbidden_edge.test(&src, &target) { - panic!( - "forbidden edge {:?} -> {:?} created", - src, target - ) - } - } - } - } - } else if cfg!(debug_assertions) { - data.current.total_duplicate_read_count.fetch_add(1, Relaxed); - } - }), + let mut task_deps = match task_deps { + TaskDepsRef::Allow(deps) => deps.lock(), TaskDepsRef::EvalAlways => { // We don't need to record dependencies of eval_always // queries. They are re-evaluated unconditionally anyway. @@ -512,6 +478,41 @@ impl DepGraph { panic!("Illegal read of: {dep_node_index:?}") } }; + let task_deps = &mut *task_deps; + + if cfg!(debug_assertions) { + data.current.total_read_count.fetch_add(1, Relaxed); + } + + // As long as we only have a low number of reads we can avoid doing a hash + // insert and potentially allocating/reallocating the hashmap + let new_read = if task_deps.reads.len() < TASK_DEPS_READS_CAP { + task_deps.reads.iter().all(|other| *other != dep_node_index) + } else { + task_deps.read_set.insert(dep_node_index) + }; + if new_read { + task_deps.reads.push(dep_node_index); + if task_deps.reads.len() == TASK_DEPS_READS_CAP { + // Fill `read_set` with what we have so far so we can use the hashset + // next time + task_deps.read_set.extend(task_deps.reads.iter().copied()); + } + + #[cfg(debug_assertions)] + { + if let Some(target) = task_deps.node { + if let Some(ref forbidden_edge) = data.current.forbidden_edge { + let src = forbidden_edge.index_to_node.lock()[&dep_node_index]; + if forbidden_edge.test(&src, &target) { + panic!("forbidden edge {:?} -> {:?} created", src, target) + } + } + } + } + } else if cfg!(debug_assertions) { + data.current.total_duplicate_read_count.fetch_add(1, Relaxed); + } }) } } @@ -573,9 +574,7 @@ impl DepGraph { let mut edges = SmallVec::new(); K::read_deps(|task_deps| match task_deps { - TaskDepsRef::Allow(deps) => { - deps.with_borrow(|deps| edges.extend(deps.reads.iter().copied())) - } + TaskDepsRef::Allow(deps) => edges.extend(deps.lock().reads.iter().copied()), TaskDepsRef::EvalAlways => { edges.push(DepNodeIndex::FOREVER_RED_NODE); } @@ -628,7 +627,7 @@ impl DepGraphData { #[inline] pub fn dep_node_index_of_opt(&self, dep_node: &DepNode) -> Option { if let Some(prev_index) = self.previous.node_to_index_opt(dep_node) { - self.current.prev_index_to_index.with_borrow(|nodes| nodes[prev_index]) + self.current.prev_index_to_index.lock()[prev_index] } else { self.current .new_node_to_index @@ -668,7 +667,7 @@ impl DepGraphData { } pub fn mark_debug_loaded_from_disk(&self, dep_node: DepNode) { - self.debug_loaded_from_disk.with_lock(|node| node.insert(dep_node)); + self.debug_loaded_from_disk.lock().insert(dep_node); } } @@ -691,29 +690,25 @@ impl DepGraph { } pub fn debug_was_loaded_from_disk(&self, dep_node: DepNode) -> bool { - self.data - .as_ref() - .unwrap() - .debug_loaded_from_disk - .with_borrow(|node| node.contains(&dep_node)) + self.data.as_ref().unwrap().debug_loaded_from_disk.lock().contains(&dep_node) } #[inline(always)] pub fn register_dep_node_debug_str(&self, dep_node: DepNode, debug_str_gen: F) - where - F: FnOnce() -> String, + where + F: FnOnce() -> String, { let dep_node_debug = &self.data.as_ref().unwrap().dep_node_debug; - if dep_node_debug.with_borrow(|node| node.contains_key(&dep_node)) { + if dep_node_debug.borrow().contains_key(&dep_node) { return; } let debug_str = self.with_ignore(debug_str_gen); - dep_node_debug.with_lock(|node| node.insert(dep_node, debug_str)); + dep_node_debug.borrow_mut().insert(dep_node, debug_str); } pub fn dep_node_debug_str(&self, dep_node: DepNode) -> Option { - self.data.as_ref()?.dep_node_debug.with_borrow(|node| node.get(&dep_node).cloned()) + self.data.as_ref()?.dep_node_debug.borrow().get(&dep_node).cloned() } fn node_color(&self, dep_node: &DepNode) -> Option { @@ -1301,26 +1296,25 @@ impl CurrentDepGraph { ) -> DepNodeIndex { self.debug_assert_not_in_new_nodes(prev_graph, prev_index); - self.prev_index_to_index.with_lock(|prev_index_to_index| { - match prev_index_to_index[prev_index] { - Some(dep_node_index) => dep_node_index, - None => { - let key = prev_graph.index_to_node(prev_index); - let edges = prev_graph - .edge_targets_from(prev_index) - .iter() - .map(|i| prev_index_to_index[*i].unwrap()) - .collect(); - let fingerprint = prev_graph.fingerprint_by_index(prev_index); - let dep_node_index = - self.encoder.borrow().send(profiler, key, fingerprint, edges); - prev_index_to_index[prev_index] = Some(dep_node_index); - #[cfg(debug_assertions)] - self.record_edge(dep_node_index, key, fingerprint); - dep_node_index - } + let mut prev_index_to_index = self.prev_index_to_index.lock(); + + match prev_index_to_index[prev_index] { + Some(dep_node_index) => dep_node_index, + None => { + let key = prev_graph.index_to_node(prev_index); + let edges = prev_graph + .edge_targets_from(prev_index) + .iter() + .map(|i| prev_index_to_index[*i].unwrap()) + .collect(); + let fingerprint = prev_graph.fingerprint_by_index(prev_index); + let dep_node_index = self.encoder.borrow().send(profiler, key, fingerprint, edges); + prev_index_to_index[prev_index] = Some(dep_node_index); + #[cfg(debug_assertions)] + self.record_edge(dep_node_index, key, fingerprint); + dep_node_index } - }) + } } #[inline] diff --git a/compiler/rustc_query_system/src/dep_graph/serialized.rs b/compiler/rustc_query_system/src/dep_graph/serialized.rs index 65b64562e42b7..edddfda624242 100644 --- a/compiler/rustc_query_system/src/dep_graph/serialized.rs +++ b/compiler/rustc_query_system/src/dep_graph/serialized.rs @@ -186,7 +186,7 @@ impl EncoderState { if let Some(record_graph) = &mut record_graph.try_lock() { record_graph.push(index, node.node, &node.edges); } - }; + } if let Some(stats) = &mut self.stats { let kind = node.node.kind; @@ -242,7 +242,7 @@ impl> GraphEncoder { pub(crate) fn with_query(&self, f: impl Fn(&DepGraphQuery)) { if let Some(record_graph) = &self.record_graph { - record_graph.with_borrow(f) + f(&record_graph.lock()) } } @@ -307,7 +307,7 @@ impl> GraphEncoder { ) -> DepNodeIndex { let _prof_timer = profiler.generic_activity("incr_comp_encode_dep_graph"); let node = NodeInfo { node, fingerprint, edges }; - self.status.with_lock(|status| status.encode_node(&node, &self.record_graph)) + self.status.lock().encode_node(&node, &self.record_graph) } pub fn finish(self, profiler: &SelfProfilerRef) -> FileEncodeResult { diff --git a/compiler/rustc_query_system/src/ich/impls_syntax.rs b/compiler/rustc_query_system/src/ich/impls_syntax.rs index feedc46e1b31b..8865ecf3e054a 100644 --- a/compiler/rustc_query_system/src/ich/impls_syntax.rs +++ b/compiler/rustc_query_system/src/ich/impls_syntax.rs @@ -80,7 +80,7 @@ impl<'a> HashStable> for SourceFile { src_hash.hash_stable(hcx, hasher); // We are always in `Lines` form by the time we reach here. - assert!(self.lines.with_borrow(|lines| lines.is_lines())); + assert!(self.lines.borrow().is_lines()); self.lines(|lines| { // We only hash the relative position within this source_file lines.len().hash_stable(hcx, hasher); diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 060da8ada795e..5976b9aa3e74a 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -1748,7 +1748,7 @@ options! { /// in the future. Note that -Zthreads=0 is the way to get /// the num_cpus behavior. #[rustc_lint_opt_deny_field_access("use `Session::threads` instead of this field")] - threads: usize = (8, parse_threads, [UNTRACKED], + threads: usize = (1, parse_threads, [UNTRACKED], "use a thread pool with N threads"), time_llvm_passes: bool = (false, parse_bool, [UNTRACKED], "measure time of each LLVM pass (default: no)"), diff --git a/compiler/rustc_session/src/parse.rs b/compiler/rustc_session/src/parse.rs index 5ca5dbb12177c..5cc9c62617dd5 100644 --- a/compiler/rustc_session/src/parse.rs +++ b/compiler/rustc_session/src/parse.rs @@ -39,7 +39,7 @@ impl GatedSpans { /// Feature gate the given `span` under the given `feature` /// which is same `Symbol` used in `active.rs`. pub fn gate(&self, feature: Symbol, span: Span) { - self.spans.with_lock(|spans| spans.entry(feature).or_default().push(span)); + self.spans.borrow_mut().entry(feature).or_default().push(span); } /// Ungate the last span under the given `feature`. @@ -47,8 +47,7 @@ impl GatedSpans { /// /// Using this is discouraged unless you have a really good reason to. pub fn ungate_last(&self, feature: Symbol, span: Span) { - let removed_span = - self.spans.with_lock(|spans| spans.entry(feature).or_default().pop().unwrap()); + let removed_span = self.spans.borrow_mut().entry(feature).or_default().pop().unwrap(); debug_assert_eq!(span, removed_span); } @@ -56,17 +55,16 @@ impl GatedSpans { /// /// Using this is discouraged unless you have a really good reason to. pub fn is_ungated(&self, feature: Symbol) -> bool { - self.spans.with_borrow(|spans| spans.get(&feature).map_or(true, |spans| spans.is_empty())) + self.spans.borrow().get(&feature).map_or(true, |spans| spans.is_empty()) } /// Prepend the given set of `spans` onto the set in `self`. pub fn merge(&self, mut spans: FxHashMap>) { - self.spans.with_lock(|inner| { - for (gate, mut gate_spans) in inner.drain() { - spans.entry(gate).or_default().append(&mut gate_spans); - } - *inner = spans; - }) + let mut inner = self.spans.borrow_mut(); + for (gate, mut gate_spans) in inner.drain() { + spans.entry(gate).or_default().append(&mut gate_spans); + } + *inner = spans; } } @@ -80,9 +78,7 @@ impl SymbolGallery { /// Insert a symbol and its span into symbol gallery. /// If the symbol has occurred before, ignore the new occurrence. pub fn insert(&self, symbol: Symbol, span: Span) { - self.symbols.with_lock(|symbols| { - symbols.entry(symbol).or_insert(span); - }) + self.symbols.lock().entry(symbol).or_insert(span); } }