From 882a9684f9bc94afa7c97fdb5724511cd04e16fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Sun, 14 May 2023 21:53:05 +0200 Subject: [PATCH] Specialize query execution for incremental and non-incremental --- compiler/rustc_interface/src/passes.rs | 3 ++ compiler/rustc_query_impl/src/lib.rs | 6 ++- compiler/rustc_query_impl/src/plumbing.rs | 38 +++++++++++++-- .../rustc_query_system/src/query/plumbing.rs | 46 +++++++++++++++---- 4 files changed, 76 insertions(+), 17 deletions(-) diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index d3af01474b85f..6fd9500307205 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -691,6 +691,8 @@ pub fn create_global_ctxt<'tcx>( callback(sess, &mut local_providers, &mut extern_providers); } + let incremental = dep_graph.is_fully_enabled(); + sess.time("setup_global_ctxt", || { gcx_cell.get_or_init(move || { TyCtxt::create_global_ctxt( @@ -705,6 +707,7 @@ pub fn create_global_ctxt<'tcx>( local_providers, extern_providers, query_result_on_disk_cache, + incremental, ), ) }) diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 39ca465954175..5a058bbfd9d73 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -34,7 +34,8 @@ use rustc_middle::ty::TyCtxt; use rustc_query_system::dep_graph::SerializedDepNodeIndex; use rustc_query_system::ich::StableHashingContext; use rustc_query_system::query::{ - get_query, HashResult, QueryCache, QueryConfig, QueryInfo, QueryMap, QueryMode, QueryState, + get_query_incr, get_query_non_incr, HashResult, QueryCache, QueryConfig, QueryInfo, QueryMap, + QueryMode, QueryState, }; use rustc_query_system::HandleCycleError; use rustc_query_system::Value; @@ -203,6 +204,7 @@ pub fn query_system<'tcx>( local_providers: Providers, extern_providers: ExternProviders, on_disk_cache: Option>, + incremental: bool, ) -> QuerySystem<'tcx> { QuerySystem { states: Default::default(), @@ -211,7 +213,7 @@ pub fn query_system<'tcx>( dynamic_queries: dynamic_queries(), on_disk_cache, fns: QuerySystemFns { - engine: engine(), + engine: engine(incremental), local_providers, extern_providers, query_structs: make_dep_kind_array!(query_structs).to_vec(), diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index 74924e8113e6f..5acfeb47a12ad 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -494,7 +494,7 @@ macro_rules! define_queries { ( $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => { - mod get_query { + mod get_query_incr { use super::*; $( @@ -506,7 +506,7 @@ macro_rules! define_queries { key: query_keys::$name<'tcx>, mode: QueryMode, ) -> Option>> { - get_query( + get_query_incr( queries::$name::config(tcx), QueryCtxt::new(tcx), span, @@ -517,9 +517,37 @@ macro_rules! define_queries { )* } - pub(crate) fn engine() -> QueryEngine { - QueryEngine { - $($name: get_query::$name,)* + mod get_query_non_incr { + use super::*; + + $( + #[inline(always)] + #[tracing::instrument(level = "trace", skip(tcx))] + pub(super) fn $name<'tcx>( + tcx: TyCtxt<'tcx>, + span: Span, + key: query_keys::$name<'tcx>, + __mode: QueryMode, + ) -> Option>> { + Some(get_query_non_incr( + queries::$name::config(tcx), + QueryCtxt::new(tcx), + span, + key, + )) + } + )* + } + + pub(crate) fn engine(incremental: bool) -> QueryEngine { + if incremental { + QueryEngine { + $($name: get_query_incr::$name,)* + } + } else { + QueryEngine { + $($name: get_query_non_incr::$name,)* + } } } diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 4aaedc7a6c15e..dbfe62ae6e943 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -312,7 +312,7 @@ where } #[inline(never)] -fn try_execute_query( +fn try_execute_query( query: Q, qcx: Qcx, span: Span, @@ -355,7 +355,7 @@ where // Drop the lock before we start executing the query drop(state_lock); - execute_job(query, qcx, state, key, id, dep_node) + execute_job::<_, _, INCR>(query, qcx, state, key, id, dep_node) } Entry::Occupied(mut entry) => { match entry.get_mut() { @@ -383,7 +383,7 @@ where } #[inline(always)] -fn execute_job( +fn execute_job( query: Q, qcx: Qcx, state: &QueryState, @@ -398,9 +398,19 @@ where // Use `JobOwner` so the query will be poisoned if executing it panics. let job_owner = JobOwner { state, key }; - let (result, dep_node_index) = match qcx.dep_context().dep_graph().data() { - None => execute_job_non_incr(query, qcx, key, id), - Some(data) => execute_job_incr(query, qcx, data, key, dep_node, id), + debug_assert_eq!(qcx.dep_context().dep_graph().is_fully_enabled(), INCR); + + let (result, dep_node_index) = if INCR { + execute_job_incr( + query, + qcx, + qcx.dep_context().dep_graph().data().unwrap(), + key, + dep_node, + id, + ) + } else { + execute_job_non_incr(query, qcx, key, id) }; let cache = query.query_cache(qcx); @@ -784,7 +794,18 @@ pub enum QueryMode { } #[inline(always)] -pub fn get_query( +pub fn get_query_non_incr(query: Q, qcx: Qcx, span: Span, key: Q::Key) -> Q::Value +where + Q: QueryConfig, + Qcx: QueryContext, +{ + debug_assert!(!qcx.dep_context().dep_graph().is_fully_enabled()); + + ensure_sufficient_stack(|| try_execute_query::(query, qcx, span, key, None).0) +} + +#[inline(always)] +pub fn get_query_incr( query: Q, qcx: Qcx, span: Span, @@ -795,6 +816,8 @@ where Q: QueryConfig, Qcx: QueryContext, { + debug_assert!(qcx.dep_context().dep_graph().is_fully_enabled()); + let dep_node = if let QueryMode::Ensure { check_cache } = mode { let (must_run, dep_node) = ensure_must_run(query, qcx, &key, check_cache); if !must_run { @@ -805,8 +828,9 @@ where None }; - let (result, dep_node_index) = - ensure_sufficient_stack(|| try_execute_query(query, qcx, span, key, dep_node)); + let (result, dep_node_index) = ensure_sufficient_stack(|| { + try_execute_query::<_, _, true>(query, qcx, span, key, dep_node) + }); if let Some(dep_node_index) = dep_node_index { qcx.dep_context().dep_graph().read_index(dep_node_index) } @@ -831,5 +855,7 @@ pub fn force_query( debug_assert!(!query.anon()); - ensure_sufficient_stack(|| try_execute_query(query, qcx, DUMMY_SP, key, Some(dep_node))); + ensure_sufficient_stack(|| { + try_execute_query::<_, _, true>(query, qcx, DUMMY_SP, key, Some(dep_node)) + }); }