diff --git a/compiler/rustc_macros/src/query.rs b/compiler/rustc_macros/src/query.rs index e7c054653accb..79b25fa5691b8 100644 --- a/compiler/rustc_macros/src/query.rs +++ b/compiler/rustc_macros/src/query.rs @@ -500,11 +500,11 @@ pub fn rustc_queries(input: TokenStream) -> TokenStream { ::rustc_middle::dep_graph::DepKind::#name => { if <#arg as DepNodeParams>>::can_reconstruct_query_key() { if let Some(key) = <#arg as DepNodeParams>>::recover($tcx, $dep_node) { - force_query::, _>( + call_query::, _>( $tcx, - key, DUMMY_SP, - *$dep_node + key, + QueryCaller::Force(*$dep_node), ); return true; } diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs index 1a8aacc486986..64523cc751ac9 100644 --- a/compiler/rustc_middle/src/ty/query/plumbing.rs +++ b/compiler/rustc_middle/src/ty/query/plumbing.rs @@ -338,7 +338,7 @@ macro_rules! define_queries_inner { // HACK(eddyb) this is like the `impl QueryConfig for queries::$name` // below, but using type aliases instead of associated types, to bypass // the limitations around normalizing under HRTB - for example, this: - // `for<'tcx> fn(...) -> as QueryConfig>>::Value` + // `for<'tcx> fn(...) -> as QueryConfig>::Value` // doesn't currently normalize to `for<'tcx> fn(...) -> query_values::$name<'tcx>`. // This is primarily used by the `provide!` macro in `rustc_metadata`. #[allow(nonstandard_style, unused_lifetimes)] @@ -413,7 +413,12 @@ macro_rules! define_queries_inner { $($(#[$attr])* #[inline(always)] pub fn $name(self, key: query_helper_param_ty!($($K)*)) { - ensure_query::, _>(self.tcx, key.into_query_param()) + call_query::, _>( + self.tcx, + DUMMY_SP, + key.into_query_param(), + QueryCaller::Ensure, + ); })* } @@ -496,7 +501,13 @@ macro_rules! define_queries_inner { pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> as QueryConfig>::Stored { - get_query::, _>(self.tcx, self.span, key.into_query_param()) + let ret = call_query::, _>( + self.tcx, + self.span, + key.into_query_param(), + QueryCaller::Get, + ); + ret.unwrap() })* } diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 50f443716f44b..278cf17eec05c 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -14,12 +14,13 @@ use rustc_data_structures::cold_path; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::{FxHashMap, FxHasher}; use rustc_data_structures::sharded::Sharded; +use rustc_data_structures::stable_hasher::HashStable; use rustc_data_structures::sync::{Lock, LockGuard}; use rustc_data_structures::thin_vec::ThinVec; use rustc_errors::{Diagnostic, FatalError}; -use rustc_span::source_map::DUMMY_SP; use rustc_span::Span; use std::collections::hash_map::Entry; +use std::fmt::Debug; use std::hash::{Hash, Hasher}; use std::mem; use std::num::NonZeroU32; @@ -158,7 +159,7 @@ where impl<'tcx, D, Q, C> JobOwner<'tcx, D, Q, C> where - D: Copy + Clone + Eq + Hash, + D: Copy + Clone + Eq + Hash + DepKind, Q: Clone, C: QueryCache, { @@ -180,7 +181,8 @@ where query: &QueryVtable, ) -> TryGetJob<'b, CTX::DepKind, CTX::Query, C> where - CTX: QueryContext, + CTX: QueryContext, + Q: HashStable, { let lock = &mut *lookup.lock; @@ -394,43 +396,29 @@ where } #[inline(always)] -fn try_execute_query( +fn try_execute_query( tcx: CTX, - state: &QueryState, - span: Span, - key: C::Key, - lookup: QueryLookup<'_, CTX::DepKind, CTX::Query, C::Key, C::Sharded>, - query: &QueryVtable, -) -> C::Stored + job_id: QueryJobId, + key: K, + query: &QueryVtable, +) -> (V, DepNodeIndex, bool) where - C: QueryCache, - C::Key: crate::dep_graph::DepNodeParams, + K: Eq + Clone + Debug + crate::dep_graph::DepNodeParams, CTX: QueryContext, { - let job = match JobOwner::<'_, CTX::DepKind, CTX::Query, C>::try_start( - tcx, state, span, &key, lookup, query, - ) { - TryGetJob::NotYetStarted(job) => job, - TryGetJob::Cycle(result) => return result, - #[cfg(parallel_compiler)] - TryGetJob::JobCompleted((v, index)) => { - tcx.dep_graph().read_index(index); - return v; - } - }; - // Fast path for when incr. comp. is off. `to_dep_node` is // expensive for some `DepKind`s. if !tcx.dep_graph().is_fully_enabled() { let null_dep_node = DepNode::new_no_params(DepKind::NULL); - return force_query_with_job(tcx, key, job, null_dep_node, query).0; + let (result, dep_node_index) = force_query_with_job(tcx, key, job_id, null_dep_node, query); + return (result, dep_node_index, false); } if query.anon { let prof_timer = tcx.profiler().query_provider(); let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| { - tcx.start_query(job.id, diagnostics, |tcx| { + tcx.start_query(job_id, diagnostics, |tcx| { tcx.dep_graph().with_anon_task(query.dep_kind, || query.compute(tcx, key)) }) }); @@ -443,7 +431,7 @@ where tcx.store_diagnostics_for_anon_node(dep_node_index, diagnostics); } - return job.complete(result, dep_node_index); + return (result, dep_node_index, false); } let dep_node = query.to_dep_node(tcx, &key); @@ -452,30 +440,26 @@ where // The diagnostics for this query will be // promoted to the current session during // `try_mark_green()`, so we can ignore them here. - let loaded = tcx.start_query(job.id, None, |tcx| { - let marked = tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node); - marked.map(|(prev_dep_node_index, dep_node_index)| { - ( - load_from_disk_and_cache_in_memory( - tcx, - key.clone(), - prev_dep_node_index, - dep_node_index, - &dep_node, - query, - ), - dep_node_index, - ) - }) + let loaded = tcx.start_query(job_id, None, |tcx| { + let (prev_dep_node_index, dep_node_index) = + tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node)?; + let result = load_from_disk_and_cache_in_memory( + tcx, + key.clone(), + prev_dep_node_index, + dep_node_index, + &dep_node, + query, + ); + Some((result, dep_node_index)) }); if let Some((result, dep_node_index)) = loaded { - return job.complete(result, dep_node_index); + return (result, dep_node_index, false); } } - let (result, dep_node_index) = force_query_with_job(tcx, key, job, dep_node, query); - tcx.dep_graph().read_index(dep_node_index); - result + let (result, dep_node_index) = force_query_with_job(tcx, key, job_id, dep_node, query); + return (result, dep_node_index, true); } fn load_from_disk_and_cache_in_memory( @@ -567,15 +551,15 @@ fn incremental_verify_ich( } #[inline(always)] -fn force_query_with_job( +fn force_query_with_job( tcx: CTX, - key: C::Key, - job: JobOwner<'_, CTX::DepKind, CTX::Query, C>, + key: K, + job_id: QueryJobId, dep_node: DepNode, - query: &QueryVtable, -) -> (C::Stored, DepNodeIndex) + query: &QueryVtable, +) -> (V, DepNodeIndex) where - C: QueryCache, + K: Debug, CTX: QueryContext, { // If the following assertion triggers, it can have two reasons: @@ -595,7 +579,7 @@ where let prof_timer = tcx.profiler().query_provider(); let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| { - tcx.start_query(job.id, diagnostics, |tcx| { + tcx.start_query(job_id, diagnostics, |tcx| { if query.eval_always { tcx.dep_graph().with_eval_always_task( dep_node, @@ -618,63 +602,32 @@ where } } - let result = job.complete(result, dep_node_index); - (result, dep_node_index) } -#[inline(never)] -fn get_query_impl( - tcx: CTX, - state: &QueryState, - span: Span, - key: C::Key, - query: &QueryVtable, -) -> C::Stored -where - CTX: QueryContext, - C: QueryCache, - C::Key: crate::dep_graph::DepNodeParams, -{ - try_get_cached( - tcx, - state, - key, - |value, index| { - tcx.dep_graph().read_index(index); - value.clone() - }, - |key, lookup| try_execute_query(tcx, state, span, key, lookup, query), - ) -} - /// Ensure that either this query has all green inputs or been executed. /// Executing `query::ensure(D)` is considered a read of the dep-node `D`. /// /// This function is particularly useful when executing passes for their /// side-effects -- e.g., in order to report errors for erroneous programs. /// +/// Return `true` if the query has already been executed. +/// /// Note: The optimization is only available during incr. comp. #[inline(never)] -fn ensure_query_impl( - tcx: CTX, - state: &QueryState, - key: C::Key, - query: &QueryVtable, -) where - C: QueryCache, - C::Key: crate::dep_graph::DepNodeParams, +fn ensure_query_impl(tcx: CTX, key: &K, query: &QueryVtable) -> bool +where + K: crate::dep_graph::DepNodeParams, CTX: QueryContext, { if query.eval_always { - let _ = get_query_impl(tcx, state, DUMMY_SP, key, query); - return; + return false; } // Ensuring an anonymous query makes no sense assert!(!query.anon); - let dep_node = query.to_dep_node(tcx, &key); + let dep_node = query.to_dep_node(tcx, key); match tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node) { None => { @@ -684,79 +637,114 @@ fn ensure_query_impl( // DepNodeIndex. We must invoke the query itself. The performance cost // this introduces should be negligible as we'll immediately hit the // in-memory cache, or another query down the line will. - let _ = get_query_impl(tcx, state, DUMMY_SP, key, query); + false } Some((_, dep_node_index)) => { tcx.profiler().query_cache_hit(dep_node_index.into()); + true } } } +pub enum QueryCaller { + Ensure, + Get, + Force(DepNode), +} + #[inline(never)] -fn force_query_impl( +fn call_query_impl( tcx: CTX, state: &QueryState, - key: C::Key, span: Span, - dep_node: DepNode, + key: C::Key, + caller: QueryCaller, query: &QueryVtable, -) where +) -> Option +where C: QueryCache, - C::Key: crate::dep_graph::DepNodeParams, + C::Key: Eq + Clone + crate::dep_graph::DepNodeParams, + C::Stored: Clone, CTX: QueryContext, { - // We may be concurrently trying both execute and force a query. - // Ensure that only one of them runs the query. + if let QueryCaller::Ensure = caller { + if ensure_query_impl(tcx, &key, query) { + return None; + } + } try_get_cached( tcx, state, key, - |_, _| { - // Cache hit, do nothing + |value, index| { + match &caller { + QueryCaller::Ensure => { + tcx.dep_graph().read_index(index); + None + } + QueryCaller::Get => { + tcx.dep_graph().read_index(index); + Some(value.clone()) + } + QueryCaller::Force(_) => { + // Cache hit, do nothing + None + } + } }, |key, lookup| { - let job = match JobOwner::<'_, CTX::DepKind, CTX::Query, C>::try_start( - tcx, state, span, &key, lookup, query, - ) { + let job = JobOwner::try_start(tcx, state, span, &key, lookup, query); + + if let QueryCaller::Force(dep_node) = &caller { + // We may be concurrently trying both execute and force a query. + // Ensure that only one of them runs the query. + + let job = match job { + TryGetJob::NotYetStarted(job) => job, + TryGetJob::Cycle(_) => return None, + #[cfg(parallel_compiler)] + TryGetJob::JobCompleted(_) => { + return None; + } + }; + let (result, dep_node_index) = + force_query_with_job(tcx, key, job.id, *dep_node, query); + return Some(job.complete(result, dep_node_index)); + }; + + let job = match job { TryGetJob::NotYetStarted(job) => job, - TryGetJob::Cycle(_) => return, + TryGetJob::Cycle(result) => return Some(result), #[cfg(parallel_compiler)] - TryGetJob::JobCompleted(_) => return, + TryGetJob::JobCompleted((v, index)) => { + tcx.dep_graph().read_index(index); + return Some(v); + } }; - force_query_with_job(tcx, key, job, dep_node, query); + + let (result, dep_node_index, read_index) = try_execute_query(tcx, job.id, key, query); + if read_index { + tcx.dep_graph().read_index(dep_node_index); + } + let result = job.complete(result, dep_node_index); + Some(result) }, - ); + ) } #[inline(always)] -pub fn get_query(tcx: CTX, span: Span, key: Q::Key) -> Q::Stored +pub fn call_query( + tcx: CTX, + span: Span, + key: Q::Key, + caller: QueryCaller, +) -> Option where Q: QueryDescription, Q::Key: crate::dep_graph::DepNodeParams, CTX: QueryContext, { debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span); - - get_query_impl(tcx, Q::query_state(tcx), span, key, &Q::VTABLE) -} - -#[inline(always)] -pub fn ensure_query(tcx: CTX, key: Q::Key) -where - Q: QueryDescription, - Q::Key: crate::dep_graph::DepNodeParams, - CTX: QueryContext, -{ - ensure_query_impl(tcx, Q::query_state(tcx), key, &Q::VTABLE) -} - -#[inline(always)] -pub fn force_query(tcx: CTX, key: Q::Key, span: Span, dep_node: DepNode) -where - Q: QueryDescription, - Q::Key: crate::dep_graph::DepNodeParams, - CTX: QueryContext, -{ - force_query_impl(tcx, Q::query_state(tcx), key, span, dep_node, &Q::VTABLE) + call_query_impl(tcx, Q::query_state(tcx), span, key, caller, &Q::VTABLE) }