From addc880b1fbcc2b6d90668bb208b41b5ea2717a1 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Mon, 5 Oct 2020 20:08:11 +0200 Subject: [PATCH 01/10] Merge {get,ensure,force}_query into call_query. --- compiler/rustc_macros/src/query.rs | 6 +- .../rustc_middle/src/ty/query/plumbing.rs | 30 +++++++++- .../rustc_query_system/src/query/plumbing.rs | 55 +++++++++++++------ 3 files changed, 68 insertions(+), 23 deletions(-) diff --git a/compiler/rustc_macros/src/query.rs b/compiler/rustc_macros/src/query.rs index e7c054653accb..98022e9c6a25b 100644 --- a/compiler/rustc_macros/src/query.rs +++ b/compiler/rustc_macros/src/query.rs @@ -500,11 +500,11 @@ pub fn rustc_queries(input: TokenStream) -> TokenStream { ::rustc_middle::dep_graph::DepKind::#name => { if <#arg as DepNodeParams>>::can_reconstruct_query_key() { if let Some(key) = <#arg as DepNodeParams>>::recover($tcx, $dep_node) { - force_query::, _>( + crate::ty::query::queries::#name::query( $tcx, - key, DUMMY_SP, - *$dep_node + key, + QueryCaller::Force(*$dep_node), ); return true; } diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs index 1a8aacc486986..5c774c403deac 100644 --- a/compiler/rustc_middle/src/ty/query/plumbing.rs +++ b/compiler/rustc_middle/src/ty/query/plumbing.rs @@ -338,7 +338,7 @@ macro_rules! define_queries_inner { // HACK(eddyb) this is like the `impl QueryConfig for queries::$name` // below, but using type aliases instead of associated types, to bypass // the limitations around normalizing under HRTB - for example, this: - // `for<'tcx> fn(...) -> as QueryConfig>>::Value` + // `for<'tcx> fn(...) -> as QueryConfig>::Value` // doesn't currently normalize to `for<'tcx> fn(...) -> query_values::$name<'tcx>`. // This is primarily used by the `provide!` macro in `rustc_metadata`. #[allow(nonstandard_style, unused_lifetimes)] @@ -402,6 +402,19 @@ macro_rules! define_queries_inner { ) -> Self::Value { handle_cycle_error!([$($modifiers)*][tcx, error]) } + } + + impl queries::$name<$tcx> { + $(#[$attr])* + #[inline(always)] + pub fn query( + tcx: TyCtxt<$tcx>, + span: Span, + key: query_keys::$name<$tcx>, + caller: QueryCaller, + ) -> Option< as QueryConfig>::Stored> { + call_query::, _>(tcx, span, key, caller) + } })* #[derive(Copy, Clone)] @@ -413,7 +426,12 @@ macro_rules! define_queries_inner { $($(#[$attr])* #[inline(always)] pub fn $name(self, key: query_helper_param_ty!($($K)*)) { - ensure_query::, _>(self.tcx, key.into_query_param()) + queries::$name::query( + self.tcx, + DUMMY_SP, + key.into_query_param(), + QueryCaller::Ensure, + ); })* } @@ -496,7 +514,13 @@ macro_rules! define_queries_inner { pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> as QueryConfig>::Stored { - get_query::, _>(self.tcx, self.span, key.into_query_param()) + let ret = queries::$name::query( + self.tcx, + self.span, + key.into_query_param(), + QueryCaller::Get, + ); + ret.unwrap() })* } diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 50f443716f44b..daf1cf73e515b 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -729,34 +729,55 @@ fn force_query_impl( ); } -#[inline(always)] -pub fn get_query(tcx: CTX, span: Span, key: Q::Key) -> Q::Stored -where - Q: QueryDescription, - Q::Key: crate::dep_graph::DepNodeParams, - CTX: QueryContext, -{ - debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span); - - get_query_impl(tcx, Q::query_state(tcx), span, key, &Q::VTABLE) +pub enum QueryCaller { + Ensure, + Get, + Force(DepNode), } -#[inline(always)] -pub fn ensure_query(tcx: CTX, key: Q::Key) +#[inline(never)] +fn call_query_impl( + tcx: CTX, + state: &QueryState, + span: Span, + key: C::Key, + caller: QueryCaller, + query: &QueryVtable, +) -> Option where - Q: QueryDescription, - Q::Key: crate::dep_graph::DepNodeParams, + C: QueryCache, + C::Key: Eq + Clone + crate::dep_graph::DepNodeParams, + C::Stored: Clone, CTX: QueryContext, { - ensure_query_impl(tcx, Q::query_state(tcx), key, &Q::VTABLE) + match caller { + QueryCaller::Ensure => { + ensure_query_impl(tcx, state, key, query); + None + } + QueryCaller::Get => { + let ret = get_query_impl(tcx, state, span, key, query); + Some(ret) + } + QueryCaller::Force(dep_node) => { + force_query_impl(tcx, state, key, span, dep_node, query); + None + } + } } #[inline(always)] -pub fn force_query(tcx: CTX, key: Q::Key, span: Span, dep_node: DepNode) +pub fn call_query( + tcx: CTX, + span: Span, + key: Q::Key, + caller: QueryCaller, +) -> Option where Q: QueryDescription, Q::Key: crate::dep_graph::DepNodeParams, CTX: QueryContext, { - force_query_impl(tcx, Q::query_state(tcx), key, span, dep_node, &Q::VTABLE) + debug!("ty::query::get_query<{}>(key={:?}, span={:?})", Q::NAME, key, span); + call_query_impl(tcx, Q::query_state(tcx), span, key, caller, &Q::VTABLE) } From 3e23b7650bd13c028c7ca1470ab14c5753807860 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sun, 11 Oct 2020 18:01:26 +0200 Subject: [PATCH 02/10] Move get_query_impl out of ensure_query_impl. --- .../rustc_query_system/src/query/plumbing.rs | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index daf1cf73e515b..2bcec5edf94fd 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -17,7 +17,6 @@ use rustc_data_structures::sharded::Sharded; use rustc_data_structures::sync::{Lock, LockGuard}; use rustc_data_structures::thin_vec::ThinVec; use rustc_errors::{Diagnostic, FatalError}; -use rustc_span::source_map::DUMMY_SP; use rustc_span::Span; use std::collections::hash_map::Entry; use std::hash::{Hash, Hasher}; @@ -656,25 +655,19 @@ where /// /// Note: The optimization is only available during incr. comp. #[inline(never)] -fn ensure_query_impl( - tcx: CTX, - state: &QueryState, - key: C::Key, - query: &QueryVtable, -) where - C: QueryCache, - C::Key: crate::dep_graph::DepNodeParams, +fn ensure_query_impl(tcx: CTX, key: &K, query: &QueryVtable) -> bool +where + K: crate::dep_graph::DepNodeParams, CTX: QueryContext, { if query.eval_always { - let _ = get_query_impl(tcx, state, DUMMY_SP, key, query); - return; + return false; } // Ensuring an anonymous query makes no sense assert!(!query.anon); - let dep_node = query.to_dep_node(tcx, &key); + let dep_node = query.to_dep_node(tcx, key); match tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node) { None => { @@ -684,10 +677,11 @@ fn ensure_query_impl( // DepNodeIndex. We must invoke the query itself. The performance cost // this introduces should be negligible as we'll immediately hit the // in-memory cache, or another query down the line will. - let _ = get_query_impl(tcx, state, DUMMY_SP, key, query); + false } Some((_, dep_node_index)) => { tcx.profiler().query_cache_hit(dep_node_index.into()); + true } } } @@ -752,7 +746,10 @@ where { match caller { QueryCaller::Ensure => { - ensure_query_impl(tcx, state, key, query); + if ensure_query_impl(tcx, &key, query) { + return None; + } + let _ = get_query_impl(tcx, state, span, key, query); None } QueryCaller::Get => { From 3f66069fcdb95ae2a8bcb530aaa2df122795f28b Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Mon, 5 Oct 2020 21:16:41 +0200 Subject: [PATCH 03/10] Remove {get,force}_query_impl. --- .../rustc_query_system/src/query/plumbing.rs | 133 +++++++----------- 1 file changed, 54 insertions(+), 79 deletions(-) diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 2bcec5edf94fd..626431af7bd02 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -14,6 +14,7 @@ use rustc_data_structures::cold_path; use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::{FxHashMap, FxHasher}; use rustc_data_structures::sharded::Sharded; +use rustc_data_structures::stable_hasher::HashStable; use rustc_data_structures::sync::{Lock, LockGuard}; use rustc_data_structures::thin_vec::ThinVec; use rustc_errors::{Diagnostic, FatalError}; @@ -157,7 +158,7 @@ where impl<'tcx, D, Q, C> JobOwner<'tcx, D, Q, C> where - D: Copy + Clone + Eq + Hash, + D: Copy + Clone + Eq + Hash + DepKind, Q: Clone, C: QueryCache, { @@ -179,7 +180,8 @@ where query: &QueryVtable, ) -> TryGetJob<'b, CTX::DepKind, CTX::Query, C> where - CTX: QueryContext, + CTX: QueryContext, + Q: HashStable, { let lock = &mut *lookup.lock; @@ -622,31 +624,6 @@ where (result, dep_node_index) } -#[inline(never)] -fn get_query_impl( - tcx: CTX, - state: &QueryState, - span: Span, - key: C::Key, - query: &QueryVtable, -) -> C::Stored -where - CTX: QueryContext, - C: QueryCache, - C::Key: crate::dep_graph::DepNodeParams, -{ - try_get_cached( - tcx, - state, - key, - |value, index| { - tcx.dep_graph().read_index(index); - value.clone() - }, - |key, lookup| try_execute_query(tcx, state, span, key, lookup, query), - ) -} - /// Ensure that either this query has all green inputs or been executed. /// Executing `query::ensure(D)` is considered a read of the dep-node `D`. /// @@ -686,43 +663,6 @@ where } } -#[inline(never)] -fn force_query_impl( - tcx: CTX, - state: &QueryState, - key: C::Key, - span: Span, - dep_node: DepNode, - query: &QueryVtable, -) where - C: QueryCache, - C::Key: crate::dep_graph::DepNodeParams, - CTX: QueryContext, -{ - // We may be concurrently trying both execute and force a query. - // Ensure that only one of them runs the query. - - try_get_cached( - tcx, - state, - key, - |_, _| { - // Cache hit, do nothing - }, - |key, lookup| { - let job = match JobOwner::<'_, CTX::DepKind, CTX::Query, C>::try_start( - tcx, state, span, &key, lookup, query, - ) { - TryGetJob::NotYetStarted(job) => job, - TryGetJob::Cycle(_) => return, - #[cfg(parallel_compiler)] - TryGetJob::JobCompleted(_) => return, - }; - force_query_with_job(tcx, key, job, dep_node, query); - }, - ); -} - pub enum QueryCaller { Ensure, Get, @@ -744,23 +684,58 @@ where C::Stored: Clone, CTX: QueryContext, { - match caller { - QueryCaller::Ensure => { - if ensure_query_impl(tcx, &key, query) { - return None; - } - let _ = get_query_impl(tcx, state, span, key, query); - None - } - QueryCaller::Get => { - let ret = get_query_impl(tcx, state, span, key, query); - Some(ret) - } - QueryCaller::Force(dep_node) => { - force_query_impl(tcx, state, key, span, dep_node, query); - None + if let QueryCaller::Ensure = caller { + if ensure_query_impl(tcx, &key, query) { + return None; } } + + try_get_cached( + tcx, + state, + key, + |value, index| { + match &caller { + QueryCaller::Ensure => { + tcx.dep_graph().read_index(index); + None + } + QueryCaller::Get => { + tcx.dep_graph().read_index(index); + Some(value.clone()) + } + QueryCaller::Force(_) => { + // Cache hit, do nothing + None + } + } + }, + |key, lookup| { + match &caller { + QueryCaller::Ensure => { + try_execute_query(tcx, state, span, key, lookup, query); + None + } + QueryCaller::Get => { + let value = try_execute_query(tcx, state, span, key, lookup, query); + Some(value) + } + QueryCaller::Force(dep_node) => { + // We may be concurrently trying both execute and force a query. + // Ensure that only one of them runs the query. + + let job = match JobOwner::try_start(tcx, state, span, &key, lookup, query) { + TryGetJob::NotYetStarted(job) => job, + TryGetJob::Cycle(_) => return None, + #[cfg(parallel_compiler)] + TryGetJob::JobCompleted(_) => return None, + }; + force_query_with_job(tcx, key, job, *dep_node, query); + None + } + } + }, + ) } #[inline(always)] From c986f6d964e5a71088a6e70e0776f70641082c16 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Mon, 5 Oct 2020 21:44:10 +0200 Subject: [PATCH 04/10] Refactor logic into try_execute_query. --- .../rustc_query_system/src/query/plumbing.rs | 48 ++++++++----------- 1 file changed, 21 insertions(+), 27 deletions(-) diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 626431af7bd02..6b9a85104d818 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -401,6 +401,7 @@ fn try_execute_query( span: Span, key: C::Key, lookup: QueryLookup<'_, CTX::DepKind, CTX::Query, C::Key, C::Sharded>, + caller: &QueryCaller, query: &QueryVtable, ) -> C::Stored where @@ -408,9 +409,26 @@ where C::Key: crate::dep_graph::DepNodeParams, CTX: QueryContext, { - let job = match JobOwner::<'_, CTX::DepKind, CTX::Query, C>::try_start( + let job = JobOwner::<'_, CTX::DepKind, CTX::Query, C>::try_start( tcx, state, span, &key, lookup, query, - ) { + ); + + if let QueryCaller::Force(dep_node) = caller { + // We may be concurrently trying both execute and force a query. + // Ensure that only one of them runs the query. + + let job = match job { + TryGetJob::NotYetStarted(job) => job, + TryGetJob::Cycle(result) => return result, + #[cfg(parallel_compiler)] + TryGetJob::JobCompleted((v, _)) => { + return v; + } + }; + return force_query_with_job(tcx, key, job, *dep_node, query).0; + }; + + let job = match job { TryGetJob::NotYetStarted(job) => job, TryGetJob::Cycle(result) => return result, #[cfg(parallel_compiler)] @@ -710,31 +728,7 @@ where } } }, - |key, lookup| { - match &caller { - QueryCaller::Ensure => { - try_execute_query(tcx, state, span, key, lookup, query); - None - } - QueryCaller::Get => { - let value = try_execute_query(tcx, state, span, key, lookup, query); - Some(value) - } - QueryCaller::Force(dep_node) => { - // We may be concurrently trying both execute and force a query. - // Ensure that only one of them runs the query. - - let job = match JobOwner::try_start(tcx, state, span, &key, lookup, query) { - TryGetJob::NotYetStarted(job) => job, - TryGetJob::Cycle(_) => return None, - #[cfg(parallel_compiler)] - TryGetJob::JobCompleted(_) => return None, - }; - force_query_with_job(tcx, key, job, *dep_node, query); - None - } - } - }, + |key, lookup| Some(try_execute_query(tcx, state, span, key, lookup, &caller, query)), ) } From 7fddbdeef390ca8235e2050b3de35808a1cb099c Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Tue, 6 Oct 2020 22:57:58 +0200 Subject: [PATCH 05/10] Move job.complete out of force_query_with_job. --- .../rustc_query_system/src/query/plumbing.rs | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 6b9a85104d818..283e4b3f5f607 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -20,6 +20,7 @@ use rustc_data_structures::thin_vec::ThinVec; use rustc_errors::{Diagnostic, FatalError}; use rustc_span::Span; use std::collections::hash_map::Entry; +use std::fmt::Debug; use std::hash::{Hash, Hasher}; use std::mem; use std::num::NonZeroU32; @@ -425,7 +426,8 @@ where return v; } }; - return force_query_with_job(tcx, key, job, *dep_node, query).0; + let (result, dep_node_index) = force_query_with_job(tcx, key, job.id, *dep_node, query); + return job.complete(result, dep_node_index); }; let job = match job { @@ -442,7 +444,8 @@ where // expensive for some `DepKind`s. if !tcx.dep_graph().is_fully_enabled() { let null_dep_node = DepNode::new_no_params(DepKind::NULL); - return force_query_with_job(tcx, key, job, null_dep_node, query).0; + let (result, dep_node_index) = force_query_with_job(tcx, key, job.id, null_dep_node, query); + return job.complete(result, dep_node_index); } if query.anon { @@ -492,7 +495,8 @@ where } } - let (result, dep_node_index) = force_query_with_job(tcx, key, job, dep_node, query); + let (result, dep_node_index) = force_query_with_job(tcx, key, job.id, dep_node, query); + let result = job.complete(result, dep_node_index); tcx.dep_graph().read_index(dep_node_index); result } @@ -586,15 +590,15 @@ fn incremental_verify_ich( } #[inline(always)] -fn force_query_with_job( +fn force_query_with_job( tcx: CTX, - key: C::Key, - job: JobOwner<'_, CTX::DepKind, CTX::Query, C>, + key: K, + job_id: QueryJobId, dep_node: DepNode, - query: &QueryVtable, -) -> (C::Stored, DepNodeIndex) + query: &QueryVtable, +) -> (V, DepNodeIndex) where - C: QueryCache, + K: Eq + Clone + Debug, CTX: QueryContext, { // If the following assertion triggers, it can have two reasons: @@ -614,7 +618,7 @@ where let prof_timer = tcx.profiler().query_provider(); let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| { - tcx.start_query(job.id, diagnostics, |tcx| { + tcx.start_query(job_id, diagnostics, |tcx| { if query.eval_always { tcx.dep_graph().with_eval_always_task( dep_node, @@ -637,8 +641,6 @@ where } } - let result = job.complete(result, dep_node_index); - (result, dep_node_index) } From d4bf8fb2c143ef242ce55ee8b8b845c6bcfe9288 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Tue, 6 Oct 2020 23:09:37 +0200 Subject: [PATCH 06/10] Move job creation/completion out of try_execute_query. --- .../rustc_query_system/src/query/plumbing.rs | 102 +++++++++--------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 283e4b3f5f607..317f3e5fbe2f1 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -396,63 +396,29 @@ where } #[inline(always)] -fn try_execute_query( +fn try_execute_query( tcx: CTX, - state: &QueryState, - span: Span, - key: C::Key, - lookup: QueryLookup<'_, CTX::DepKind, CTX::Query, C::Key, C::Sharded>, - caller: &QueryCaller, - query: &QueryVtable, -) -> C::Stored + job_id: QueryJobId, + key: K, + query: &QueryVtable, +) -> (V, DepNodeIndex, bool) where - C: QueryCache, - C::Key: crate::dep_graph::DepNodeParams, + K: Eq + Clone + Debug + crate::dep_graph::DepNodeParams, CTX: QueryContext, { - let job = JobOwner::<'_, CTX::DepKind, CTX::Query, C>::try_start( - tcx, state, span, &key, lookup, query, - ); - - if let QueryCaller::Force(dep_node) = caller { - // We may be concurrently trying both execute and force a query. - // Ensure that only one of them runs the query. - - let job = match job { - TryGetJob::NotYetStarted(job) => job, - TryGetJob::Cycle(result) => return result, - #[cfg(parallel_compiler)] - TryGetJob::JobCompleted((v, _)) => { - return v; - } - }; - let (result, dep_node_index) = force_query_with_job(tcx, key, job.id, *dep_node, query); - return job.complete(result, dep_node_index); - }; - - let job = match job { - TryGetJob::NotYetStarted(job) => job, - TryGetJob::Cycle(result) => return result, - #[cfg(parallel_compiler)] - TryGetJob::JobCompleted((v, index)) => { - tcx.dep_graph().read_index(index); - return v; - } - }; - // Fast path for when incr. comp. is off. `to_dep_node` is // expensive for some `DepKind`s. if !tcx.dep_graph().is_fully_enabled() { let null_dep_node = DepNode::new_no_params(DepKind::NULL); - let (result, dep_node_index) = force_query_with_job(tcx, key, job.id, null_dep_node, query); - return job.complete(result, dep_node_index); + let (result, dep_node_index) = force_query_with_job(tcx, key, job_id, null_dep_node, query); + return (result, dep_node_index, false); } if query.anon { let prof_timer = tcx.profiler().query_provider(); let ((result, dep_node_index), diagnostics) = with_diagnostics(|diagnostics| { - tcx.start_query(job.id, diagnostics, |tcx| { + tcx.start_query(job_id, diagnostics, |tcx| { tcx.dep_graph().with_anon_task(query.dep_kind, || query.compute(tcx, key)) }) }); @@ -465,7 +431,7 @@ where tcx.store_diagnostics_for_anon_node(dep_node_index, diagnostics); } - return job.complete(result, dep_node_index); + return (result, dep_node_index, false); } let dep_node = query.to_dep_node(tcx, &key); @@ -474,7 +440,7 @@ where // The diagnostics for this query will be // promoted to the current session during // `try_mark_green()`, so we can ignore them here. - let loaded = tcx.start_query(job.id, None, |tcx| { + let loaded = tcx.start_query(job_id, None, |tcx| { let marked = tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node); marked.map(|(prev_dep_node_index, dep_node_index)| { ( @@ -491,14 +457,12 @@ where }) }); if let Some((result, dep_node_index)) = loaded { - return job.complete(result, dep_node_index); + return (result, dep_node_index, false); } } - let (result, dep_node_index) = force_query_with_job(tcx, key, job.id, dep_node, query); - let result = job.complete(result, dep_node_index); - tcx.dep_graph().read_index(dep_node_index); - result + let (result, dep_node_index) = force_query_with_job(tcx, key, job_id, dep_node, query); + return (result, dep_node_index, true); } fn load_from_disk_and_cache_in_memory( @@ -730,7 +694,43 @@ where } } }, - |key, lookup| Some(try_execute_query(tcx, state, span, key, lookup, &caller, query)), + |key, lookup| { + let job = JobOwner::try_start(tcx, state, span, &key, lookup, query); + + if let QueryCaller::Force(dep_node) = &caller { + // We may be concurrently trying both execute and force a query. + // Ensure that only one of them runs the query. + + let job = match job { + TryGetJob::NotYetStarted(job) => job, + TryGetJob::Cycle(_) => return None, + #[cfg(parallel_compiler)] + TryGetJob::JobCompleted(_) => { + return None; + } + }; + let (result, dep_node_index) = + force_query_with_job(tcx, key, job.id, *dep_node, query); + return Some(job.complete(result, dep_node_index)); + }; + + let job = match job { + TryGetJob::NotYetStarted(job) => job, + TryGetJob::Cycle(result) => return Some(result), + #[cfg(parallel_compiler)] + TryGetJob::JobCompleted((v, index)) => { + tcx.dep_graph().read_index(index); + return Some(v); + } + }; + + let (result, dep_node_index, read_index) = try_execute_query(tcx, job.id, key, query); + if read_index { + tcx.dep_graph().read_index(dep_node_index); + } + let result = job.complete(result, dep_node_index); + Some(result) + }, ) } From e256d2262cc7f58e8e41fb8caddbda3122a7a9a1 Mon Sep 17 00:00:00 2001 From: Julian Wollersberger Date: Tue, 13 Oct 2020 14:47:16 +0200 Subject: [PATCH 07/10] Removed the wrapper around `call_query()`. --- compiler/rustc_macros/src/query.rs | 2 +- compiler/rustc_middle/src/ty/query/plumbing.rs | 17 ++--------------- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/compiler/rustc_macros/src/query.rs b/compiler/rustc_macros/src/query.rs index 98022e9c6a25b..79b25fa5691b8 100644 --- a/compiler/rustc_macros/src/query.rs +++ b/compiler/rustc_macros/src/query.rs @@ -500,7 +500,7 @@ pub fn rustc_queries(input: TokenStream) -> TokenStream { ::rustc_middle::dep_graph::DepKind::#name => { if <#arg as DepNodeParams>>::can_reconstruct_query_key() { if let Some(key) = <#arg as DepNodeParams>>::recover($tcx, $dep_node) { - crate::ty::query::queries::#name::query( + call_query::, _>( $tcx, DUMMY_SP, key, diff --git a/compiler/rustc_middle/src/ty/query/plumbing.rs b/compiler/rustc_middle/src/ty/query/plumbing.rs index 5c774c403deac..64523cc751ac9 100644 --- a/compiler/rustc_middle/src/ty/query/plumbing.rs +++ b/compiler/rustc_middle/src/ty/query/plumbing.rs @@ -402,19 +402,6 @@ macro_rules! define_queries_inner { ) -> Self::Value { handle_cycle_error!([$($modifiers)*][tcx, error]) } - } - - impl queries::$name<$tcx> { - $(#[$attr])* - #[inline(always)] - pub fn query( - tcx: TyCtxt<$tcx>, - span: Span, - key: query_keys::$name<$tcx>, - caller: QueryCaller, - ) -> Option< as QueryConfig>::Stored> { - call_query::, _>(tcx, span, key, caller) - } })* #[derive(Copy, Clone)] @@ -426,7 +413,7 @@ macro_rules! define_queries_inner { $($(#[$attr])* #[inline(always)] pub fn $name(self, key: query_helper_param_ty!($($K)*)) { - queries::$name::query( + call_query::, _>( self.tcx, DUMMY_SP, key.into_query_param(), @@ -514,7 +501,7 @@ macro_rules! define_queries_inner { pub fn $name(self, key: query_helper_param_ty!($($K)*)) -> as QueryConfig>::Stored { - let ret = queries::$name::query( + let ret = call_query::, _>( self.tcx, self.span, key.into_query_param(), From 945d588845e02a63c5f10fe6013242760deda826 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Wed, 14 Oct 2020 20:12:05 +0200 Subject: [PATCH 08/10] Add comment. --- compiler/rustc_query_system/src/query/plumbing.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 317f3e5fbe2f1..893f74fa01e67 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -614,6 +614,8 @@ where /// This function is particularly useful when executing passes for their /// side-effects -- e.g., in order to report errors for erroneous programs. /// +/// Return `true` if the query has already been executed. +/// /// Note: The optimization is only available during incr. comp. #[inline(never)] fn ensure_query_impl(tcx: CTX, key: &K, query: &QueryVtable) -> bool From 4d9e839137a535c90489f17544605488b4060bef Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Wed, 14 Oct 2020 22:04:48 +0200 Subject: [PATCH 09/10] Remove useless bounds. --- compiler/rustc_query_system/src/query/plumbing.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 893f74fa01e67..07a0f497a2208 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -562,7 +562,7 @@ fn force_query_with_job( query: &QueryVtable, ) -> (V, DepNodeIndex) where - K: Eq + Clone + Debug, + K: Debug, CTX: QueryContext, { // If the following assertion triggers, it can have two reasons: From 99d65b0d495b7ae07608cf6229397eed892cbb84 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Wed, 14 Oct 2020 22:05:00 +0200 Subject: [PATCH 10/10] Simplify forcing. --- .../rustc_query_system/src/query/plumbing.rs | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 07a0f497a2208..278cf17eec05c 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -441,20 +441,17 @@ where // promoted to the current session during // `try_mark_green()`, so we can ignore them here. let loaded = tcx.start_query(job_id, None, |tcx| { - let marked = tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node); - marked.map(|(prev_dep_node_index, dep_node_index)| { - ( - load_from_disk_and_cache_in_memory( - tcx, - key.clone(), - prev_dep_node_index, - dep_node_index, - &dep_node, - query, - ), - dep_node_index, - ) - }) + let (prev_dep_node_index, dep_node_index) = + tcx.dep_graph().try_mark_green_and_read(tcx, &dep_node)?; + let result = load_from_disk_and_cache_in_memory( + tcx, + key.clone(), + prev_dep_node_index, + dep_node_index, + &dep_node, + query, + ); + Some((result, dep_node_index)) }); if let Some((result, dep_node_index)) = loaded { return (result, dep_node_index, false);