From a51a20531d7ff532486ff73abb9ad548618481ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Wed, 8 Feb 2023 19:53:48 +0100 Subject: [PATCH 1/5] Factor query arena allocation out from query caches --- compiler/rustc_interface/src/passes.rs | 6 +- .../src/rmeta/decoder/cstore_impl.rs | 2 +- compiler/rustc_middle/src/ty/context.rs | 8 +- compiler/rustc_middle/src/ty/query.rs | 96 ++++++++-- compiler/rustc_query_impl/src/lib.rs | 6 +- compiler/rustc_query_impl/src/plumbing.rs | 65 ++++--- .../rustc_query_system/src/query/caches.rs | 171 +----------------- .../rustc_query_system/src/query/config.rs | 9 +- .../rustc_query_system/src/query/plumbing.rs | 49 +++-- 9 files changed, 159 insertions(+), 253 deletions(-) diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index 6a94d19001e11..0eff576c92640 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -676,9 +676,7 @@ pub fn create_global_ctxt<'tcx>( callback(sess, &mut local_providers, &mut extern_providers); } - let queries = queries.get_or_init(|| { - TcxQueries::new(local_providers, extern_providers, query_result_on_disk_cache) - }); + let queries = queries.get_or_init(|| TcxQueries::new(query_result_on_disk_cache)); sess.time("setup_global_ctxt", || { gcx_cell.get_or_init(move || { @@ -690,6 +688,8 @@ pub fn create_global_ctxt<'tcx>( untracked, dep_graph, queries.on_disk_cache.as_ref().map(OnDiskCache::as_dyn), + local_providers, + extern_providers, queries.as_dyn(), rustc_query_impl::query_callbacks(arena), ) diff --git a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs index 8082a89032047..0bacf51e9119a 100644 --- a/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs +++ b/compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs @@ -114,7 +114,7 @@ macro_rules! provide_one { fn $name<'tcx>( $tcx: TyCtxt<'tcx>, def_id_arg: ty::query::query_keys::$name<'tcx>, - ) -> ty::query::query_values::$name<'tcx> { + ) -> ty::query::query_provided::$name<'tcx> { let _prof_timer = $tcx.prof.generic_activity(concat!("metadata_decode_entry_", stringify!($name))); diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index 4aef071cd9827..e334d3395f5ac 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -18,6 +18,8 @@ use crate::mir::{ use crate::thir::Thir; use crate::traits; use crate::traits::solve::{ExternalConstraints, ExternalConstraintsData}; +use crate::ty::query::ExternProviders; +use crate::ty::query::Providers; use crate::ty::query::{self, TyCtxtAt}; use crate::ty::{ self, AdtDef, AdtDefData, AdtKind, Binder, Const, ConstData, DefIdTree, FloatTy, FloatVar, @@ -479,7 +481,7 @@ pub struct GlobalCtxt<'tcx> { pub on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>, pub queries: &'tcx dyn query::QueryEngine<'tcx>, - pub query_caches: query::QueryCaches<'tcx>, + pub query_system: query::QuerySystem<'tcx>, pub(crate) query_kinds: &'tcx [DepKindStruct<'tcx>], // Internal caches for metadata decoding. No need to track deps on this. @@ -639,6 +641,8 @@ impl<'tcx> TyCtxt<'tcx> { untracked: Untracked, dep_graph: DepGraph, on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>, + local_providers: Providers, + extern_providers: ExternProviders, queries: &'tcx dyn query::QueryEngine<'tcx>, query_kinds: &'tcx [DepKindStruct<'tcx>], ) -> GlobalCtxt<'tcx> { @@ -664,7 +668,7 @@ impl<'tcx> TyCtxt<'tcx> { untracked, on_disk_cache, queries, - query_caches: query::QueryCaches::default(), + query_system: query::QuerySystem::new(local_providers, extern_providers), query_kinds, ty_rcache: Default::default(), pred_rcache: Default::default(), diff --git a/compiler/rustc_middle/src/ty/query.rs b/compiler/rustc_middle/src/ty/query.rs index ed54aa96f5b89..73e4a39681e7f 100644 --- a/compiler/rustc_middle/src/ty/query.rs +++ b/compiler/rustc_middle/src/ty/query.rs @@ -1,3 +1,5 @@ +#![allow(unused_parens)] + use crate::dep_graph; use crate::infer::canonical::{self, Canonical}; use crate::lint::LintExpectation; @@ -34,6 +36,7 @@ use crate::ty::subst::{GenericArg, SubstsRef}; use crate::ty::util::AlwaysRequiresDrop; use crate::ty::GeneratorDiagnosticData; use crate::ty::{self, CrateInherentImpls, ParamEnvAnd, Ty, TyCtxt, UnusedGenericParams}; +use rustc_arena::TypedArena; use rustc_ast as ast; use rustc_ast::expand::allocator::AllocatorKind; use rustc_attr as attr; @@ -41,6 +44,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexMap, FxIndexSet}; use rustc_data_structures::steal::Steal; use rustc_data_structures::svh::Svh; use rustc_data_structures::sync::Lrc; +use rustc_data_structures::sync::WorkerLocal; use rustc_data_structures::unord::UnordSet; use rustc_errors::ErrorGuaranteed; use rustc_hir as hir; @@ -66,6 +70,24 @@ use std::sync::Arc; pub(crate) use rustc_query_system::query::QueryJobId; use rustc_query_system::query::*; +pub struct QuerySystem<'tcx> { + pub local_providers: Box, + pub extern_providers: Box, + pub arenas: QueryArenas<'tcx>, + pub caches: QueryCaches<'tcx>, +} + +impl<'tcx> QuerySystem<'tcx> { + pub fn new(local_providers: Providers, extern_providers: ExternProviders) -> Self { + QuerySystem { + local_providers: Box::new(local_providers), + extern_providers: Box::new(extern_providers), + arenas: Default::default(), + caches: Default::default(), + } + } +} + #[derive(Copy, Clone)] pub struct TyCtxtAt<'tcx> { pub tcx: TyCtxt<'tcx>, @@ -112,10 +134,10 @@ macro_rules! query_helper_param_ty { } macro_rules! query_if_arena { - ([] $arena:ty, $no_arena:ty) => { + ([] $arena:tt $no_arena:tt) => { $no_arena }; - ([(arena_cache) $($rest:tt)*] $arena:ty, $no_arena:ty) => { + ([(arena_cache) $($rest:tt)*] $arena:tt $no_arena:tt) => { $arena }; ([$other:tt $($modifiers:tt)*]$($args:tt)*) => { @@ -131,7 +153,7 @@ macro_rules! separate_provide_extern_decl { for<'tcx> fn( TyCtxt<'tcx>, query_keys::$name<'tcx>, - ) -> query_values::$name<'tcx> + ) -> query_provided::$name<'tcx> }; ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => { separate_provide_extern_decl!([$($modifiers)*][$($args)*]) @@ -183,30 +205,62 @@ macro_rules! define_callbacks { $(pub type $name<'tcx> = $($K)*;)* } - #[allow(nonstandard_style, unused_lifetimes, unused_parens)] + #[allow(nonstandard_style, unused_lifetimes)] pub mod query_values { use super::*; - $(pub type $name<'tcx> = query_if_arena!([$($modifiers)*] <$V as Deref>::Target, $V);)* + $(pub type $name<'tcx> = $V;)* } - #[allow(nonstandard_style, unused_lifetimes, unused_parens)] - pub mod query_storage { + #[allow(nonstandard_style, unused_lifetimes)] + pub mod query_provided { use super::*; $( - pub type $name<'tcx> = query_if_arena!([$($modifiers)*] - <<$($K)* as Key>::CacheSelector - as CacheSelector<'tcx, <$V as Deref>::Target>>::ArenaCache, - <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache - ); + pub type $name<'tcx> = query_if_arena!([$($modifiers)*] (<$V as Deref>::Target) ($V)); )* } + #[allow(nonstandard_style, unused_lifetimes)] + pub mod query_provided_to_value { + use super::*; + $( + #[inline] + pub fn $name<'tcx>( + _tcx: TyCtxt<'tcx>, + value: query_provided::$name<'tcx>, + ) -> query_values::$name<'tcx> { + query_if_arena!([$($modifiers)*] + (&*_tcx.query_system.arenas.$name.alloc(value)) + (value) + ) + } + )* + } #[allow(nonstandard_style, unused_lifetimes)] - pub mod query_stored { + pub mod query_storage { use super::*; - $(pub type $name<'tcx> = $V;)* + $( + pub type $name<'tcx> = <<$($K)* as Key>::CacheSelector as CacheSelector<'tcx, $V>>::Cache; + )* + } + + pub struct QueryArenas<'tcx> { + $($(#[$attr])* pub $name: query_if_arena!([$($modifiers)*] + (WorkerLocal::Target>>) + () + ),)* + } + + impl Default for QueryArenas<'_> { + fn default() -> Self { + Self { + $($name: query_if_arena!([$($modifiers)*] + (WorkerLocal::new(|_| Default::default())) + () + ),)* + } + } } #[derive(Default)] @@ -221,7 +275,7 @@ macro_rules! define_callbacks { let key = key.into_query_param(); opt_remap_env_constness!([$($modifiers)*][key]); - match try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key) { + match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) { Some(_) => return, None => self.tcx.queries.$name(self.tcx, DUMMY_SP, key, QueryMode::Ensure), }; @@ -246,7 +300,7 @@ macro_rules! define_callbacks { let key = key.into_query_param(); opt_remap_env_constness!([$($modifiers)*][key]); - match try_get_cached(self.tcx, &self.tcx.query_caches.$name, &key) { + match try_get_cached(self.tcx, &self.tcx.query_system.caches.$name, &key) { Some(value) => value, None => self.tcx.queries.$name(self.tcx, self.span, key, QueryMode::Get).unwrap(), } @@ -257,7 +311,7 @@ macro_rules! define_callbacks { $(pub $name: for<'tcx> fn( TyCtxt<'tcx>, query_keys::$name<'tcx>, - ) -> query_values::$name<'tcx>,)* + ) -> query_provided::$name<'tcx>,)* } pub struct ExternProviders { @@ -334,12 +388,13 @@ macro_rules! define_feedable { $(impl<'tcx, K: IntoQueryParam<$($K)*> + Copy> TyCtxtFeed<'tcx, K> { $(#[$attr])* #[inline(always)] - pub fn $name(self, value: query_values::$name<'tcx>) -> $V { + pub fn $name(self, value: query_provided::$name<'tcx>) -> $V { let key = self.key().into_query_param(); opt_remap_env_constness!([$($modifiers)*][key]); let tcx = self.tcx; - let cache = &tcx.query_caches.$name; + let value = query_provided_to_value::$name(tcx, value); + let cache = &tcx.query_system.caches.$name; match try_get_cached(tcx, cache, &key) { Some(old) => { @@ -357,7 +412,8 @@ macro_rules! define_feedable { &value, hash_result!([$($modifiers)*]), ); - cache.complete(key, value, dep_node_index) + cache.complete(key, value, dep_node_index); + value } } } diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 2d243e13cc212..372f2c69c7813 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -21,8 +21,10 @@ use rustc_data_structures::sync::AtomicU64; use rustc_middle::arena::Arena; use rustc_middle::dep_graph::{self, DepKindStruct}; use rustc_middle::query::Key; -use rustc_middle::ty::query::{query_keys, query_storage, query_stored, query_values}; -use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine}; +use rustc_middle::ty::query::QueryEngine; +use rustc_middle::ty::query::{ + query_keys, query_provided, query_provided_to_value, query_storage, query_values, +}; use rustc_middle::ty::TyCtxt; use rustc_span::Span; diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index 49309db564ea8..318077e75c4fa 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -278,13 +278,13 @@ macro_rules! hash_result { macro_rules! get_provider { ([][$tcx:expr, $name:ident, $key:expr]) => {{ - $tcx.queries.local_providers.$name + $tcx.query_system.local_providers.$name }}; ([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{ if $key.query_crate_is_local() { - $tcx.queries.local_providers.$name + $tcx.query_system.local_providers.$name } else { - $tcx.queries.extern_providers.$name + $tcx.query_system.extern_providers.$name } }}; ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => { @@ -293,14 +293,14 @@ macro_rules! get_provider { } macro_rules! should_ever_cache_on_disk { - ([]) => {{ - None + ([]$yes:tt $no:tt) => {{ + $no }}; - ([(cache) $($rest:tt)*]) => {{ - Some($crate::plumbing::try_load_from_disk::) + ([(cache) $($rest:tt)*]$yes:tt $no:tt) => {{ + $yes }}; - ([$other:tt $($modifiers:tt)*]) => { - should_ever_cache_on_disk!([$($modifiers)*]) + ([$other:tt $($modifiers:tt)*]$yes:tt $no:tt) => { + should_ever_cache_on_disk!([$($modifiers)*]$yes $no) }; } @@ -472,7 +472,6 @@ macro_rules! define_queries { $(impl<'tcx> QueryConfig> for queries::$name<'tcx> { type Key = query_keys::$name<'tcx>; type Value = query_values::$name<'tcx>; - type Stored = query_stored::$name<'tcx>; const NAME: &'static str = stringify!($name); #[inline] @@ -493,24 +492,40 @@ macro_rules! define_queries { fn query_cache<'a>(tcx: QueryCtxt<'tcx>) -> &'a Self::Cache where 'tcx:'a { - &tcx.query_caches.$name + &tcx.query_system.caches.$name } - fn execute_query(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Stored { + fn execute_query(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value { tcx.$name(key) } #[inline] // key is only sometimes used #[allow(unused_variables)] - fn compute(qcx: QueryCtxt<'tcx>, key: &Self::Key) -> fn(TyCtxt<'tcx>, Self::Key) -> Self::Value { - get_provider!([$($modifiers)*][qcx, $name, key]) + fn compute(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value { + query_provided_to_value::$name( + tcx, + get_provider!([$($modifiers)*][tcx, $name, key])(tcx, key) + ) } #[inline] - fn try_load_from_disk(qcx: QueryCtxt<'tcx>, key: &Self::Key) -> rustc_query_system::query::TryLoadFromDisk, Self> { - let cache_on_disk = Self::cache_on_disk(qcx.tcx, key); - if cache_on_disk { should_ever_cache_on_disk!([$($modifiers)*]) } else { None } + fn try_load_from_disk(_qcx: QueryCtxt<'tcx>, _key: &Self::Key) -> rustc_query_system::query::TryLoadFromDisk, Self> { + should_ever_cache_on_disk!([$($modifiers)*] { + if Self::cache_on_disk(_qcx.tcx, _key) { + Some(|qcx: QueryCtxt<'tcx>, dep_node| { + let value = $crate::plumbing::try_load_from_disk::>( + qcx, + dep_node + ); + value.map(|value| query_provided_to_value::$name(qcx.tcx, value)) + }) + } else { + None + } + } { + None + }) } const ANON: bool = is_anon!([$($modifiers)*]); @@ -633,7 +648,7 @@ macro_rules! define_queries { $crate::profiling_support::alloc_self_profile_query_strings_for_query_cache( tcx, stringify!($name), - &tcx.query_caches.$name, + &tcx.query_system.caches.$name, string_cache, ) }, @@ -649,18 +664,12 @@ macro_rules! define_queries { } } -use crate::{ExternProviders, OnDiskCache, Providers}; +use crate::OnDiskCache; impl<'tcx> Queries<'tcx> { - pub fn new( - local_providers: Providers, - extern_providers: ExternProviders, - on_disk_cache: Option>, - ) -> Self { + pub fn new(on_disk_cache: Option>) -> Self { use crate::query_structs; Queries { - local_providers: Box::new(local_providers), - extern_providers: Box::new(extern_providers), query_structs: make_dep_kind_array!(query_structs).to_vec(), on_disk_cache, jobs: AtomicU64::new(1), @@ -674,8 +683,6 @@ macro_rules! define_queries_struct { input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => { #[derive(Default)] pub struct Queries<'tcx> { - local_providers: Box, - extern_providers: Box, query_structs: Vec<$crate::plumbing::QueryStruct<'tcx>>, pub on_disk_cache: Option>, jobs: AtomicU64, @@ -725,7 +732,7 @@ macro_rules! define_queries_struct { span: Span, key: as QueryConfig>>::Key, mode: QueryMode, - ) -> Option> { + ) -> Option> { let qcx = QueryCtxt { tcx, queries: self }; get_query::, _, rustc_middle::dep_graph::DepKind>(qcx, span, key, mode) })* diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs index 81c7e4673d41b..e840108bdd869 100644 --- a/compiler/rustc_query_system/src/query/caches.rs +++ b/compiler/rustc_query_system/src/query/caches.rs @@ -1,12 +1,10 @@ use crate::dep_graph::DepNodeIndex; -use rustc_arena::TypedArena; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::sharded; #[cfg(parallel_compiler)] use rustc_data_structures::sharded::Sharded; use rustc_data_structures::sync::Lock; -use rustc_data_structures::sync::WorkerLocal; use rustc_index::vec::{Idx, IndexVec}; use std::fmt::Debug; use std::hash::Hash; @@ -16,12 +14,10 @@ pub trait CacheSelector<'tcx, V> { type Cache where V: Copy; - type ArenaCache; } pub trait QueryStorage { - type Value: Debug; - type Stored: Copy; + type Value: Copy; } pub trait QueryCache: QueryStorage + Sized { @@ -31,9 +27,9 @@ pub trait QueryCache: QueryStorage + Sized { /// It returns the shard index and a lock guard to the shard, /// which will be used if the query is not in the cache and we need /// to compute it. - fn lookup(&self, key: &Self::Key) -> Option<(Self::Stored, DepNodeIndex)>; + fn lookup(&self, key: &Self::Key) -> Option<(Self::Value, DepNodeIndex)>; - fn complete(&self, key: Self::Key, value: Self::Value, index: DepNodeIndex) -> Self::Stored; + fn complete(&self, key: Self::Key, value: Self::Value, index: DepNodeIndex); fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)); } @@ -44,7 +40,6 @@ impl<'tcx, K: Eq + Hash, V: 'tcx> CacheSelector<'tcx, V> for DefaultCacheSelecto type Cache = DefaultCache where V: Copy; - type ArenaCache = ArenaCache<'tcx, K, V>; } pub struct DefaultCache { @@ -62,7 +57,6 @@ impl Default for DefaultCache { impl QueryStorage for DefaultCache { type Value = V; - type Stored = V; } impl QueryCache for DefaultCache @@ -85,7 +79,7 @@ where } #[inline] - fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored { + fn complete(&self, key: K, value: V, index: DepNodeIndex) { #[cfg(parallel_compiler)] let mut lock = self.cache.get_shard_by_value(&key).lock(); #[cfg(not(parallel_compiler))] @@ -93,7 +87,6 @@ where // We may be overwriting another value. This is all right, since the dep-graph // will check that the fingerprint matches. lock.insert(key, (value, index)); - value } fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) { @@ -122,7 +115,6 @@ impl<'tcx, V: 'tcx> CacheSelector<'tcx, V> for SingleCacheSelector { type Cache = SingleCache where V: Copy; - type ArenaCache = ArenaCache<'tcx, (), V>; } pub struct SingleCache { @@ -137,7 +129,6 @@ impl Default for SingleCache { impl QueryStorage for SingleCache { type Value = V; - type Stored = V; } impl QueryCache for SingleCache @@ -152,9 +143,8 @@ where } #[inline] - fn complete(&self, _key: (), value: V, index: DepNodeIndex) -> Self::Stored { + fn complete(&self, _key: (), value: V, index: DepNodeIndex) { *self.cache.lock() = Some((value, index)); - value } fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) { @@ -162,85 +152,12 @@ where } } -pub struct ArenaCache<'tcx, K, V> { - arena: WorkerLocal>, - #[cfg(parallel_compiler)] - cache: Sharded>, - #[cfg(not(parallel_compiler))] - cache: Lock>, -} - -impl<'tcx, K, V> Default for ArenaCache<'tcx, K, V> { - fn default() -> Self { - ArenaCache { arena: WorkerLocal::new(|_| TypedArena::default()), cache: Default::default() } - } -} - -impl<'tcx, K: Eq + Hash, V: Debug + 'tcx> QueryStorage for ArenaCache<'tcx, K, V> { - type Value = V; - type Stored = &'tcx V; -} - -impl<'tcx, K, V: 'tcx> QueryCache for ArenaCache<'tcx, K, V> -where - K: Eq + Hash + Clone + Debug, - V: Debug, -{ - type Key = K; - - #[inline(always)] - fn lookup(&self, key: &K) -> Option<(&'tcx V, DepNodeIndex)> { - let key_hash = sharded::make_hash(key); - #[cfg(parallel_compiler)] - let lock = self.cache.get_shard_by_hash(key_hash).lock(); - #[cfg(not(parallel_compiler))] - let lock = self.cache.lock(); - let result = lock.raw_entry().from_key_hashed_nocheck(key_hash, key); - - if let Some((_, value)) = result { Some((&value.0, value.1)) } else { None } - } - - #[inline] - fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored { - let value = self.arena.alloc((value, index)); - let value = unsafe { &*(value as *const _) }; - #[cfg(parallel_compiler)] - let mut lock = self.cache.get_shard_by_value(&key).lock(); - #[cfg(not(parallel_compiler))] - let mut lock = self.cache.lock(); - // We may be overwriting another value. This is all right, since the dep-graph - // will check that the fingerprint matches. - lock.insert(key, value); - &value.0 - } - - fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) { - #[cfg(parallel_compiler)] - { - let shards = self.cache.lock_shards(); - for shard in shards.iter() { - for (k, v) in shard.iter() { - f(k, &v.0, v.1); - } - } - } - #[cfg(not(parallel_compiler))] - { - let map = self.cache.lock(); - for (k, v) in map.iter() { - f(k, &v.0, v.1); - } - } - } -} - pub struct VecCacheSelector(PhantomData); impl<'tcx, K: Idx, V: 'tcx> CacheSelector<'tcx, V> for VecCacheSelector { type Cache = VecCache where V: Copy; - type ArenaCache = VecArenaCache<'tcx, K, V>; } pub struct VecCache { @@ -258,7 +175,6 @@ impl Default for VecCache { impl QueryStorage for VecCache { type Value = V; - type Stored = V; } impl QueryCache for VecCache @@ -278,87 +194,12 @@ where } #[inline] - fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored { + fn complete(&self, key: K, value: V, index: DepNodeIndex) { #[cfg(parallel_compiler)] let mut lock = self.cache.get_shard_by_hash(key.index() as u64).lock(); #[cfg(not(parallel_compiler))] let mut lock = self.cache.lock(); lock.insert(key, (value, index)); - value - } - - fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) { - #[cfg(parallel_compiler)] - { - let shards = self.cache.lock_shards(); - for shard in shards.iter() { - for (k, v) in shard.iter_enumerated() { - if let Some(v) = v { - f(&k, &v.0, v.1); - } - } - } - } - #[cfg(not(parallel_compiler))] - { - let map = self.cache.lock(); - for (k, v) in map.iter_enumerated() { - if let Some(v) = v { - f(&k, &v.0, v.1); - } - } - } - } -} - -pub struct VecArenaCache<'tcx, K: Idx, V> { - arena: WorkerLocal>, - #[cfg(parallel_compiler)] - cache: Sharded>>, - #[cfg(not(parallel_compiler))] - cache: Lock>>, -} - -impl<'tcx, K: Idx, V> Default for VecArenaCache<'tcx, K, V> { - fn default() -> Self { - VecArenaCache { - arena: WorkerLocal::new(|_| TypedArena::default()), - cache: Default::default(), - } - } -} - -impl<'tcx, K: Eq + Idx, V: Debug + 'tcx> QueryStorage for VecArenaCache<'tcx, K, V> { - type Value = V; - type Stored = &'tcx V; -} - -impl<'tcx, K, V: 'tcx> QueryCache for VecArenaCache<'tcx, K, V> -where - K: Eq + Idx + Clone + Debug, - V: Debug, -{ - type Key = K; - - #[inline(always)] - fn lookup(&self, key: &K) -> Option<(&'tcx V, DepNodeIndex)> { - #[cfg(parallel_compiler)] - let lock = self.cache.get_shard_by_hash(key.index() as u64).lock(); - #[cfg(not(parallel_compiler))] - let lock = self.cache.lock(); - if let Some(Some(value)) = lock.get(*key) { Some((&value.0, value.1)) } else { None } - } - - #[inline] - fn complete(&self, key: K, value: V, index: DepNodeIndex) -> Self::Stored { - let value = self.arena.alloc((value, index)); - let value = unsafe { &*(value as *const _) }; - #[cfg(parallel_compiler)] - let mut lock = self.cache.get_shard_by_hash(key.index() as u64).lock(); - #[cfg(not(parallel_compiler))] - let mut lock = self.cache.lock(); - lock.insert(key, value); - &value.0 } fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) { diff --git a/compiler/rustc_query_system/src/query/config.rs b/compiler/rustc_query_system/src/query/config.rs index a28e45a5c086d..925102d4b1ead 100644 --- a/compiler/rustc_query_system/src/query/config.rs +++ b/compiler/rustc_query_system/src/query/config.rs @@ -20,10 +20,9 @@ pub trait QueryConfig { const NAME: &'static str; type Key: DepNodeParams + Eq + Hash + Clone + Debug; - type Value: Debug; - type Stored: Debug + Copy + std::borrow::Borrow; + type Value: Debug + Copy; - type Cache: QueryCache; + type Cache: QueryCache; // Don't use this method to access query results, instead use the methods on TyCtxt fn query_state<'a>(tcx: Qcx) -> &'a QueryState @@ -38,9 +37,9 @@ pub trait QueryConfig { fn cache_on_disk(tcx: Qcx::DepContext, key: &Self::Key) -> bool; // Don't use this method to compute query results, instead use the methods on TyCtxt - fn execute_query(tcx: Qcx::DepContext, k: Self::Key) -> Self::Stored; + fn execute_query(tcx: Qcx::DepContext, k: Self::Key) -> Self::Value; - fn compute(tcx: Qcx, key: &Self::Key) -> fn(Qcx::DepContext, Self::Key) -> Self::Value; + fn compute(tcx: Qcx::DepContext, key: Self::Key) -> Self::Value; fn try_load_from_disk(qcx: Qcx, idx: &Self::Key) -> TryLoadFromDisk; diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index ed66d1929c5e7..44e74844fde90 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -246,7 +246,7 @@ where /// Completes the query by updating the query cache with the `result`, /// signals the waiter and forgets the JobOwner, so it won't poison the query - fn complete(self, cache: &C, result: C::Value, dep_node_index: DepNodeIndex) -> C::Stored + fn complete(self, cache: &C, result: C::Value, dep_node_index: DepNodeIndex) where C: QueryCache, { @@ -257,23 +257,22 @@ where // Forget ourself so our destructor won't poison the query mem::forget(self); - let (job, result) = { - let job = { - #[cfg(parallel_compiler)] - let mut lock = state.active.get_shard_by_value(&key).lock(); - #[cfg(not(parallel_compiler))] - let mut lock = state.active.lock(); - match lock.remove(&key).unwrap() { - QueryResult::Started(job) => job, - QueryResult::Poisoned => panic!(), - } - }; - let result = cache.complete(key, result, dep_node_index); - (job, result) + // Mark as complete before we remove the job from the active state + // so no other thread can re-execute this query. + cache.complete(key.clone(), result, dep_node_index); + + let job = { + #[cfg(parallel_compiler)] + let mut lock = state.active.get_shard_by_value(&key).lock(); + #[cfg(not(parallel_compiler))] + let mut lock = state.active.lock(); + match lock.remove(&key).unwrap() { + QueryResult::Started(job) => job, + QueryResult::Poisoned => panic!(), + } }; job.signal_complete(); - result } } @@ -336,7 +335,7 @@ where /// which will be used if the query is not in the cache and we need /// to compute it. #[inline] -pub fn try_get_cached(tcx: Tcx, cache: &C, key: &C::Key) -> Option +pub fn try_get_cached(tcx: Tcx, cache: &C, key: &C::Key) -> Option where C: QueryCache, Tcx: DepContext, @@ -358,7 +357,7 @@ fn try_execute_query( span: Span, key: Q::Key, dep_node: Option>, -) -> (Q::Stored, Option) +) -> (Q::Value, Option) where Q: QueryConfig, Qcx: QueryContext, @@ -390,7 +389,7 @@ where ); } } - let result = job.complete(cache, result, dep_node_index); + job.complete(cache, result, dep_node_index); (result, Some(dep_node_index)) } TryGetJob::Cycle(error) => { @@ -426,9 +425,8 @@ where // Fast path for when incr. comp. is off. if !dep_graph.is_fully_enabled() { let prof_timer = qcx.dep_context().profiler().query_provider(); - let result = qcx.start_query(job_id, Q::DEPTH_LIMIT, None, || { - Q::compute(qcx, &key)(*qcx.dep_context(), key) - }); + let result = + qcx.start_query(job_id, Q::DEPTH_LIMIT, None, || Q::compute(*qcx.dep_context(), key)); let dep_node_index = dep_graph.next_virtual_depnode_index(); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); return (result, dep_node_index); @@ -455,7 +453,7 @@ where qcx.start_query(job_id, Q::DEPTH_LIMIT, Some(&diagnostics), || { if Q::ANON { return dep_graph.with_anon_task(*qcx.dep_context(), Q::DEP_KIND, || { - Q::compute(qcx, &key)(*qcx.dep_context(), key) + Q::compute(*qcx.dep_context(), key) }); } @@ -463,8 +461,7 @@ where let dep_node = dep_node_opt.unwrap_or_else(|| Q::construct_dep_node(*qcx.dep_context(), &key)); - let task = Q::compute(qcx, &key); - dep_graph.with_task(dep_node, *qcx.dep_context(), key, task, Q::HASH_RESULT) + dep_graph.with_task(dep_node, *qcx.dep_context(), key, Q::compute, Q::HASH_RESULT) }); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); @@ -555,7 +552,7 @@ where let prof_timer = qcx.dep_context().profiler().query_provider(); // The dep-graph for this computation is already in-place. - let result = dep_graph.with_ignore(|| Q::compute(qcx, key)(*qcx.dep_context(), key.clone())); + let result = dep_graph.with_ignore(|| Q::compute(*qcx.dep_context(), key.clone())); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); @@ -727,7 +724,7 @@ pub enum QueryMode { Ensure, } -pub fn get_query(qcx: Qcx, span: Span, key: Q::Key, mode: QueryMode) -> Option +pub fn get_query(qcx: Qcx, span: Span, key: Q::Key, mode: QueryMode) -> Option where D: DepKind, Q: QueryConfig, From 265e1e968d918f4dc9e62dfb712c600af136f92f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Mon, 13 Feb 2023 23:05:04 +0100 Subject: [PATCH 2/5] Use dropless arena when possible --- compiler/rustc_middle/src/ty/query.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_middle/src/ty/query.rs b/compiler/rustc_middle/src/ty/query.rs index 73e4a39681e7f..478617d055a8f 100644 --- a/compiler/rustc_middle/src/ty/query.rs +++ b/compiler/rustc_middle/src/ty/query.rs @@ -63,6 +63,7 @@ use rustc_span::symbol::Symbol; use rustc_span::{Span, DUMMY_SP}; use rustc_target::abi; use rustc_target::spec::PanicStrategy; +use std::mem; use std::ops::Deref; use std::path::PathBuf; use std::sync::Arc; @@ -230,7 +231,13 @@ macro_rules! define_callbacks { value: query_provided::$name<'tcx>, ) -> query_values::$name<'tcx> { query_if_arena!([$($modifiers)*] - (&*_tcx.query_system.arenas.$name.alloc(value)) + { + if mem::needs_drop::>() { + &*_tcx.query_system.arenas.$name.alloc(value) + } else { + &*_tcx.arena.dropless.alloc(value) + } + } (value) ) } From 067bf2ac1340a9362585d0d1690b9da24195582c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Tue, 14 Feb 2023 14:17:04 +0100 Subject: [PATCH 3/5] Move provider fields back to rustc_query_impl --- compiler/rustc_interface/src/passes.rs | 6 ++--- compiler/rustc_middle/src/ty/context.rs | 6 +---- compiler/rustc_middle/src/ty/query.rs | 14 +---------- compiler/rustc_query_impl/src/lib.rs | 2 +- compiler/rustc_query_impl/src/plumbing.rs | 25 ++++++++++++------- .../rustc_query_system/src/query/config.rs | 2 +- .../rustc_query_system/src/query/plumbing.rs | 12 ++++----- 7 files changed, 28 insertions(+), 39 deletions(-) diff --git a/compiler/rustc_interface/src/passes.rs b/compiler/rustc_interface/src/passes.rs index 0eff576c92640..6a94d19001e11 100644 --- a/compiler/rustc_interface/src/passes.rs +++ b/compiler/rustc_interface/src/passes.rs @@ -676,7 +676,9 @@ pub fn create_global_ctxt<'tcx>( callback(sess, &mut local_providers, &mut extern_providers); } - let queries = queries.get_or_init(|| TcxQueries::new(query_result_on_disk_cache)); + let queries = queries.get_or_init(|| { + TcxQueries::new(local_providers, extern_providers, query_result_on_disk_cache) + }); sess.time("setup_global_ctxt", || { gcx_cell.get_or_init(move || { @@ -688,8 +690,6 @@ pub fn create_global_ctxt<'tcx>( untracked, dep_graph, queries.on_disk_cache.as_ref().map(OnDiskCache::as_dyn), - local_providers, - extern_providers, queries.as_dyn(), rustc_query_impl::query_callbacks(arena), ) diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index e334d3395f5ac..99ad604b24189 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -18,8 +18,6 @@ use crate::mir::{ use crate::thir::Thir; use crate::traits; use crate::traits::solve::{ExternalConstraints, ExternalConstraintsData}; -use crate::ty::query::ExternProviders; -use crate::ty::query::Providers; use crate::ty::query::{self, TyCtxtAt}; use crate::ty::{ self, AdtDef, AdtDefData, AdtKind, Binder, Const, ConstData, DefIdTree, FloatTy, FloatVar, @@ -641,8 +639,6 @@ impl<'tcx> TyCtxt<'tcx> { untracked: Untracked, dep_graph: DepGraph, on_disk_cache: Option<&'tcx dyn OnDiskCache<'tcx>>, - local_providers: Providers, - extern_providers: ExternProviders, queries: &'tcx dyn query::QueryEngine<'tcx>, query_kinds: &'tcx [DepKindStruct<'tcx>], ) -> GlobalCtxt<'tcx> { @@ -668,7 +664,7 @@ impl<'tcx> TyCtxt<'tcx> { untracked, on_disk_cache, queries, - query_system: query::QuerySystem::new(local_providers, extern_providers), + query_system: Default::default(), query_kinds, ty_rcache: Default::default(), pred_rcache: Default::default(), diff --git a/compiler/rustc_middle/src/ty/query.rs b/compiler/rustc_middle/src/ty/query.rs index 478617d055a8f..ffde39b4db101 100644 --- a/compiler/rustc_middle/src/ty/query.rs +++ b/compiler/rustc_middle/src/ty/query.rs @@ -71,24 +71,12 @@ use std::sync::Arc; pub(crate) use rustc_query_system::query::QueryJobId; use rustc_query_system::query::*; +#[derive(Default)] pub struct QuerySystem<'tcx> { - pub local_providers: Box, - pub extern_providers: Box, pub arenas: QueryArenas<'tcx>, pub caches: QueryCaches<'tcx>, } -impl<'tcx> QuerySystem<'tcx> { - pub fn new(local_providers: Providers, extern_providers: ExternProviders) -> Self { - QuerySystem { - local_providers: Box::new(local_providers), - extern_providers: Box::new(extern_providers), - arenas: Default::default(), - caches: Default::default(), - } - } -} - #[derive(Copy, Clone)] pub struct TyCtxtAt<'tcx> { pub tcx: TyCtxt<'tcx>, diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs index 372f2c69c7813..d7708a3bc3f42 100644 --- a/compiler/rustc_query_impl/src/lib.rs +++ b/compiler/rustc_query_impl/src/lib.rs @@ -21,10 +21,10 @@ use rustc_data_structures::sync::AtomicU64; use rustc_middle::arena::Arena; use rustc_middle::dep_graph::{self, DepKindStruct}; use rustc_middle::query::Key; -use rustc_middle::ty::query::QueryEngine; use rustc_middle::ty::query::{ query_keys, query_provided, query_provided_to_value, query_storage, query_values, }; +use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine}; use rustc_middle::ty::TyCtxt; use rustc_span::Span; diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index 318077e75c4fa..532c1b52f5356 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -278,13 +278,13 @@ macro_rules! hash_result { macro_rules! get_provider { ([][$tcx:expr, $name:ident, $key:expr]) => {{ - $tcx.query_system.local_providers.$name + $tcx.queries.local_providers.$name }}; ([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{ if $key.query_crate_is_local() { - $tcx.query_system.local_providers.$name + $tcx.queries.local_providers.$name } else { - $tcx.query_system.extern_providers.$name + $tcx.queries.extern_providers.$name } }}; ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => { @@ -500,12 +500,11 @@ macro_rules! define_queries { } #[inline] - // key is only sometimes used #[allow(unused_variables)] - fn compute(tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value { + fn compute(qcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value { query_provided_to_value::$name( - tcx, - get_provider!([$($modifiers)*][tcx, $name, key])(tcx, key) + qcx.tcx, + get_provider!([$($modifiers)*][qcx, $name, key])(qcx.tcx, key) ) } @@ -664,12 +663,18 @@ macro_rules! define_queries { } } -use crate::OnDiskCache; +use crate::{ExternProviders, OnDiskCache, Providers}; impl<'tcx> Queries<'tcx> { - pub fn new(on_disk_cache: Option>) -> Self { + pub fn new( + local_providers: Providers, + extern_providers: ExternProviders, + on_disk_cache: Option>, + ) -> Self { use crate::query_structs; Queries { + local_providers: Box::new(local_providers), + extern_providers: Box::new(extern_providers), query_structs: make_dep_kind_array!(query_structs).to_vec(), on_disk_cache, jobs: AtomicU64::new(1), @@ -683,6 +688,8 @@ macro_rules! define_queries_struct { input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => { #[derive(Default)] pub struct Queries<'tcx> { + local_providers: Box, + extern_providers: Box, query_structs: Vec<$crate::plumbing::QueryStruct<'tcx>>, pub on_disk_cache: Option>, jobs: AtomicU64, diff --git a/compiler/rustc_query_system/src/query/config.rs b/compiler/rustc_query_system/src/query/config.rs index 925102d4b1ead..56247e827a2da 100644 --- a/compiler/rustc_query_system/src/query/config.rs +++ b/compiler/rustc_query_system/src/query/config.rs @@ -39,7 +39,7 @@ pub trait QueryConfig { // Don't use this method to compute query results, instead use the methods on TyCtxt fn execute_query(tcx: Qcx::DepContext, k: Self::Key) -> Self::Value; - fn compute(tcx: Qcx::DepContext, key: Self::Key) -> Self::Value; + fn compute(tcx: Qcx, key: Self::Key) -> Self::Value; fn try_load_from_disk(qcx: Qcx, idx: &Self::Key) -> TryLoadFromDisk; diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 44e74844fde90..38f507409c600 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -425,8 +425,7 @@ where // Fast path for when incr. comp. is off. if !dep_graph.is_fully_enabled() { let prof_timer = qcx.dep_context().profiler().query_provider(); - let result = - qcx.start_query(job_id, Q::DEPTH_LIMIT, None, || Q::compute(*qcx.dep_context(), key)); + let result = qcx.start_query(job_id, Q::DEPTH_LIMIT, None, || Q::compute(qcx, key)); let dep_node_index = dep_graph.next_virtual_depnode_index(); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); return (result, dep_node_index); @@ -452,16 +451,15 @@ where let (result, dep_node_index) = qcx.start_query(job_id, Q::DEPTH_LIMIT, Some(&diagnostics), || { if Q::ANON { - return dep_graph.with_anon_task(*qcx.dep_context(), Q::DEP_KIND, || { - Q::compute(*qcx.dep_context(), key) - }); + return dep_graph + .with_anon_task(*qcx.dep_context(), Q::DEP_KIND, || Q::compute(qcx, key)); } // `to_dep_node` is expensive for some `DepKind`s. let dep_node = dep_node_opt.unwrap_or_else(|| Q::construct_dep_node(*qcx.dep_context(), &key)); - dep_graph.with_task(dep_node, *qcx.dep_context(), key, Q::compute, Q::HASH_RESULT) + dep_graph.with_task(dep_node, qcx, key, Q::compute, Q::HASH_RESULT) }); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); @@ -552,7 +550,7 @@ where let prof_timer = qcx.dep_context().profiler().query_provider(); // The dep-graph for this computation is already in-place. - let result = dep_graph.with_ignore(|| Q::compute(*qcx.dep_context(), key.clone())); + let result = dep_graph.with_ignore(|| Q::compute(qcx, key.clone())); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); From 1ae1eac7d23dec75c34a3f2be706d2f2146748f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Tue, 14 Feb 2023 14:40:57 +0100 Subject: [PATCH 4/5] Add some comments and use inline(always) on query_provided_to_value --- compiler/rustc_middle/src/ty/query.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/compiler/rustc_middle/src/ty/query.rs b/compiler/rustc_middle/src/ty/query.rs index ffde39b4db101..5c45cc389ee64 100644 --- a/compiler/rustc_middle/src/ty/query.rs +++ b/compiler/rustc_middle/src/ty/query.rs @@ -200,6 +200,10 @@ macro_rules! define_callbacks { $(pub type $name<'tcx> = $V;)* } + + /// This module specifies the type returned from query providers and the type used for + /// decoding. For regular queries this is the declared returned type `V`, but + /// `arena_cache` will use `::Target` instead. #[allow(nonstandard_style, unused_lifetimes)] pub mod query_provided { use super::*; @@ -208,12 +212,17 @@ macro_rules! define_callbacks { pub type $name<'tcx> = query_if_arena!([$($modifiers)*] (<$V as Deref>::Target) ($V)); )* } + + /// This module has a function per query which takes a `query_provided` value and coverts + /// it to a regular `V` value by allocating it on an arena if the query has the + /// `arena_cache` modifier. This will happen when computing the query using a provider or + /// decoding a stored result. #[allow(nonstandard_style, unused_lifetimes)] pub mod query_provided_to_value { use super::*; $( - #[inline] + #[inline(always)] pub fn $name<'tcx>( _tcx: TyCtxt<'tcx>, value: query_provided::$name<'tcx>, From caf29b27277fd4fca4b372938e2fecf12dd12c78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?John=20K=C3=A5re=20Alsaker?= Date: Tue, 14 Feb 2023 15:05:10 +0100 Subject: [PATCH 5/5] Remove parallel compiler fix --- compiler/rustc_query_system/src/query/plumbing.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 38f507409c600..57217fb681a97 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -257,10 +257,6 @@ where // Forget ourself so our destructor won't poison the query mem::forget(self); - // Mark as complete before we remove the job from the active state - // so no other thread can re-execute this query. - cache.complete(key.clone(), result, dep_node_index); - let job = { #[cfg(parallel_compiler)] let mut lock = state.active.get_shard_by_value(&key).lock(); @@ -271,6 +267,7 @@ where QueryResult::Poisoned => panic!(), } }; + cache.complete(key, result, dep_node_index); job.signal_complete(); }