From 1d8489c150df7d0e751f7c438541036df1a9ac5e Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Thu, 7 May 2020 23:27:21 +0200 Subject: [PATCH 1/2] perf: Revert accidental inclusion of a part of #69218 This was accidentally included in #69494 after a rebase and given how much `inflate` and `keccak` stresses the obligation forest seems like a likely culprit to the regression in those benchmarks. (It is necessary in #69218 as obligation forest needs to accurately track the root variables or unifications will get lost) --- src/librustc_trait_selection/traits/fulfill.rs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/librustc_trait_selection/traits/fulfill.rs b/src/librustc_trait_selection/traits/fulfill.rs index 1e056c96acd38..bff1fa3312980 100644 --- a/src/librustc_trait_selection/traits/fulfill.rs +++ b/src/librustc_trait_selection/traits/fulfill.rs @@ -240,15 +240,9 @@ struct FulfillProcessor<'a, 'b, 'tcx> { register_region_obligations: bool, } -fn mk_pending( - infcx: &InferCtxt<'_, 'tcx>, - os: Vec>, -) -> Vec> { +fn mk_pending(os: Vec>) -> Vec> { os.into_iter() - .map(|mut o| { - o.predicate = infcx.resolve_vars_if_possible(&o.predicate); - PendingPredicateObligation { obligation: o, stalled_on: vec![] } - }) + .map(|o| PendingPredicateObligation { obligation: o, stalled_on: vec![] }) .collect() } @@ -342,7 +336,7 @@ impl<'a, 'b, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'b, 'tcx> { "selecting trait `{:?}` at depth {} yielded Ok(Some)", data, obligation.recursion_depth ); - ProcessResult::Changed(mk_pending(infcx, vtable.nested_obligations())) + ProcessResult::Changed(mk_pending(vtable.nested_obligations())) } Ok(None) => { debug!( @@ -436,7 +430,7 @@ impl<'a, 'b, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'b, 'tcx> { trait_ref_type_vars(self.selcx, data.to_poly_trait_ref(tcx)); ProcessResult::Unchanged } - Ok(Some(os)) => ProcessResult::Changed(mk_pending(infcx, os)), + Ok(Some(os)) => ProcessResult::Changed(mk_pending(os)), Err(e) => ProcessResult::Error(CodeProjectionError(e)), } } @@ -475,7 +469,7 @@ impl<'a, 'b, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'b, 'tcx> { vec![TyOrConstInferVar::maybe_from_ty(ty).unwrap()]; ProcessResult::Unchanged } - Some(os) => ProcessResult::Changed(mk_pending(infcx, os)), + Some(os) => ProcessResult::Changed(mk_pending(os)), } } @@ -493,7 +487,7 @@ impl<'a, 'b, 'tcx> ObligationProcessor for FulfillProcessor<'a, 'b, 'tcx> { ]; ProcessResult::Unchanged } - Some(Ok(ok)) => ProcessResult::Changed(mk_pending(infcx, ok.obligations)), + Some(Ok(ok)) => ProcessResult::Changed(mk_pending(ok.obligations)), Some(Err(err)) => { let expected_found = ExpectedFound::new( subtype.skip_binder().a_is_expected, From ebc7eda9e75829305a31a00037056a5365d261fe Mon Sep 17 00:00:00 2001 From: Markus Westerlind Date: Sun, 24 May 2020 15:22:23 +0200 Subject: [PATCH 2/2] perf: Add inline on commonly used methods added in 69464 Reclaims most of the regression in inflate --- .../snapshot_map/mod.rs | 1 + src/librustc_infer/infer/mod.rs | 16 +++++++--- .../infer/region_constraints/mod.rs | 4 +++ src/librustc_infer/infer/type_variable.rs | 31 +++++++++---------- src/librustc_infer/infer/undo_log.rs | 2 ++ src/librustc_infer/traits/project.rs | 2 ++ 6 files changed, 35 insertions(+), 21 deletions(-) diff --git a/src/librustc_data_structures/snapshot_map/mod.rs b/src/librustc_data_structures/snapshot_map/mod.rs index 52865f55f786b..b4cc85293f7c1 100644 --- a/src/librustc_data_structures/snapshot_map/mod.rs +++ b/src/librustc_data_structures/snapshot_map/mod.rs @@ -37,6 +37,7 @@ pub enum UndoLog { } impl SnapshotMap { + #[inline] pub fn with_log(&mut self, undo_log: L2) -> SnapshotMap { SnapshotMap { map: &mut self.map, undo_log, _marker: PhantomData } } diff --git a/src/librustc_infer/infer/mod.rs b/src/librustc_infer/infer/mod.rs index 2cf9dd882e4bb..5dedaced0b112 100644 --- a/src/librustc_infer/infer/mod.rs +++ b/src/librustc_infer/infer/mod.rs @@ -219,18 +219,22 @@ impl<'tcx> InferCtxtInner<'tcx> { } } + #[inline] pub fn region_obligations(&self) -> &[(hir::HirId, RegionObligation<'tcx>)] { &self.region_obligations } + #[inline] pub fn projection_cache(&mut self) -> traits::ProjectionCache<'_, 'tcx> { self.projection_cache.with_log(&mut self.undo_log) } + #[inline] fn type_variables(&mut self) -> type_variable::TypeVariableTable<'_, 'tcx> { self.type_variable_storage.with_log(&mut self.undo_log) } + #[inline] fn int_unification_table( &mut self, ) -> ut::UnificationTable< @@ -243,6 +247,7 @@ impl<'tcx> InferCtxtInner<'tcx> { self.int_unification_storage.with_log(&mut self.undo_log) } + #[inline] fn float_unification_table( &mut self, ) -> ut::UnificationTable< @@ -255,6 +260,7 @@ impl<'tcx> InferCtxtInner<'tcx> { self.float_unification_storage.with_log(&mut self.undo_log) } + #[inline] fn const_unification_table( &mut self, ) -> ut::UnificationTable< @@ -267,6 +273,7 @@ impl<'tcx> InferCtxtInner<'tcx> { self.const_unification_storage.with_log(&mut self.undo_log) } + #[inline] pub fn unwrap_region_constraints(&mut self) -> RegionConstraintCollector<'_, 'tcx> { self.region_constraint_storage .as_mut() @@ -1645,14 +1652,13 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { /// having to resort to storing full `GenericArg`s in `stalled_on`. #[inline(always)] pub fn ty_or_const_infer_var_changed(&self, infer_var: TyOrConstInferVar<'tcx>) -> bool { - let mut inner = self.inner.borrow_mut(); match infer_var { TyOrConstInferVar::Ty(v) => { use self::type_variable::TypeVariableValue; // If `inlined_probe` returns a `Known` value, it never equals // `ty::Infer(ty::TyVar(v))`. - match inner.type_variables().inlined_probe(v) { + match self.inner.borrow_mut().type_variables().inlined_probe(v) { TypeVariableValue::Unknown { .. } => false, TypeVariableValue::Known { .. } => true, } @@ -1662,7 +1668,7 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { // If `inlined_probe_value` returns a value it's always a // `ty::Int(_)` or `ty::UInt(_)`, which never matches a // `ty::Infer(_)`. - inner.int_unification_table().inlined_probe_value(v).is_some() + self.inner.borrow_mut().int_unification_table().inlined_probe_value(v).is_some() } TyOrConstInferVar::TyFloat(v) => { @@ -1670,7 +1676,7 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { // `ty::Float(_)`, which never matches a `ty::Infer(_)`. // // Not `inlined_probe_value(v)` because this call site is colder. - inner.float_unification_table().probe_value(v).is_some() + self.inner.borrow_mut().float_unification_table().probe_value(v).is_some() } TyOrConstInferVar::Const(v) => { @@ -1678,7 +1684,7 @@ impl<'a, 'tcx> InferCtxt<'a, 'tcx> { // `ty::ConstKind::Infer(ty::InferConst::Var(v))`. // // Not `inlined_probe_value(v)` because this call site is colder. - match inner.const_unification_table().probe_value(v).val { + match self.inner.borrow_mut().const_unification_table().probe_value(v).val { ConstVariableValue::Unknown { .. } => false, ConstVariableValue::Known { .. } => true, } diff --git a/src/librustc_infer/infer/region_constraints/mod.rs b/src/librustc_infer/infer/region_constraints/mod.rs index 0c9f002a2a21d..095a20105e574 100644 --- a/src/librustc_infer/infer/region_constraints/mod.rs +++ b/src/librustc_infer/infer/region_constraints/mod.rs @@ -68,12 +68,14 @@ pub struct RegionConstraintCollector<'a, 'tcx> { impl std::ops::Deref for RegionConstraintCollector<'_, 'tcx> { type Target = RegionConstraintStorage<'tcx>; + #[inline] fn deref(&self) -> &RegionConstraintStorage<'tcx> { self.storage } } impl std::ops::DerefMut for RegionConstraintCollector<'_, 'tcx> { + #[inline] fn deref_mut(&mut self) -> &mut RegionConstraintStorage<'tcx> { self.storage } @@ -345,6 +347,7 @@ impl<'tcx> RegionConstraintStorage<'tcx> { Self::default() } + #[inline] pub(crate) fn with_log<'a>( &'a mut self, undo_log: &'a mut InferCtxtUndoLogs<'tcx>, @@ -796,6 +799,7 @@ impl<'tcx> RegionConstraintCollector<'_, 'tcx> { .unwrap_or(None) } + #[inline] fn unification_table(&mut self) -> super::UnificationTable<'_, 'tcx, ty::RegionVid> { ut::UnificationTable::with_log(&mut self.storage.unification_table, self.undo_log) } diff --git a/src/librustc_infer/infer/type_variable.rs b/src/librustc_infer/infer/type_variable.rs index f68692391a288..53c7dcc637718 100644 --- a/src/librustc_infer/infer/type_variable.rs +++ b/src/librustc_infer/infer/type_variable.rs @@ -87,11 +87,7 @@ pub struct TypeVariableStorage<'tcx> { } pub struct TypeVariableTable<'a, 'tcx> { - values: &'a mut sv::SnapshotVecStorage, - - eq_relations: &'a mut ut::UnificationTableStorage>, - - sub_relations: &'a mut ut::UnificationTableStorage, + storage: &'a mut TypeVariableStorage<'tcx>, undo_log: &'a mut InferCtxtUndoLogs<'tcx>, } @@ -165,12 +161,12 @@ impl<'tcx> TypeVariableStorage<'tcx> { } } + #[inline] pub(crate) fn with_log<'a>( &'a mut self, undo_log: &'a mut InferCtxtUndoLogs<'tcx>, ) -> TypeVariableTable<'a, 'tcx> { - let TypeVariableStorage { values, eq_relations, sub_relations } = self; - TypeVariableTable { values, eq_relations, sub_relations, undo_log } + TypeVariableTable { storage: self, undo_log } } } @@ -180,7 +176,7 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> { /// Note that this function does not return care whether /// `vid` has been unified with something else or not. pub fn var_diverges(&self, vid: ty::TyVid) -> bool { - self.values.get(vid.index as usize).diverging + self.storage.values.get(vid.index as usize).diverging } /// Returns the origin that was given when `vid` was created. @@ -188,7 +184,7 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> { /// Note that this function does not return care whether /// `vid` has been unified with something else or not. pub fn var_origin(&self, vid: ty::TyVid) -> &TypeVariableOrigin { - &self.values.get(vid.index as usize).origin + &self.storage.values.get(vid.index as usize).origin } /// Records that `a == b`, depending on `dir`. @@ -265,7 +261,7 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> { /// Returns the number of type variables created thus far. pub fn num_vars(&self) -> usize { - self.values.len() + self.storage.values.len() } /// Returns the "root" variable of `vid` in the `eq_relations` @@ -319,18 +315,21 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> { } } + #[inline] fn values( &mut self, ) -> sv::SnapshotVec, &mut InferCtxtUndoLogs<'tcx>> { - self.values.with_log(self.undo_log) + self.storage.values.with_log(self.undo_log) } + #[inline] fn eq_relations(&mut self) -> super::UnificationTable<'_, 'tcx, TyVidEqKey<'tcx>> { - self.eq_relations.with_log(self.undo_log) + self.storage.eq_relations.with_log(self.undo_log) } + #[inline] fn sub_relations(&mut self) -> super::UnificationTable<'_, 'tcx, ty::TyVid> { - self.sub_relations.with_log(self.undo_log) + self.storage.sub_relations.with_log(self.undo_log) } /// Returns a range of the type variables created during the snapshot. @@ -342,7 +341,7 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> { ( range.start..range.end, (range.start.index..range.end.index) - .map(|index| self.values.get(index as usize).origin) + .map(|index| self.storage.values.get(index as usize).origin) .collect(), ) } @@ -378,7 +377,7 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> { // quick check to see if this variable was // created since the snapshot started or not. let mut eq_relations = ut::UnificationTable::with_log( - &mut *self.eq_relations, + &mut self.storage.eq_relations, &mut *self.undo_log, ); let escaping_type = match eq_relations.probe_value(vid) { @@ -400,7 +399,7 @@ impl<'tcx> TypeVariableTable<'_, 'tcx> { /// Returns indices of all variables that are not yet /// instantiated. pub fn unsolved_variables(&mut self) -> Vec { - (0..self.values.len()) + (0..self.storage.values.len()) .filter_map(|i| { let vid = ty::TyVid { index: i as u32 }; match self.probe(vid) { diff --git a/src/librustc_infer/infer/undo_log.rs b/src/librustc_infer/infer/undo_log.rs index 56cb182dbf0f9..e7f1869955d20 100644 --- a/src/librustc_infer/infer/undo_log.rs +++ b/src/librustc_infer/infer/undo_log.rs @@ -100,10 +100,12 @@ impl<'tcx, T> UndoLogs for InferCtxtUndoLogs<'tcx> where UndoLog<'tcx>: From, { + #[inline] fn num_open_snapshots(&self) -> usize { self.num_open_snapshots } + #[inline] fn push(&mut self, undo: T) { if self.in_snapshot() { self.logs.push(undo.into()) diff --git a/src/librustc_infer/traits/project.rs b/src/librustc_infer/traits/project.rs index f0d21a7d022da..65284bcee912c 100644 --- a/src/librustc_infer/traits/project.rs +++ b/src/librustc_infer/traits/project.rs @@ -95,6 +95,7 @@ pub enum ProjectionCacheEntry<'tcx> { } impl<'tcx> ProjectionCacheStorage<'tcx> { + #[inline] pub(crate) fn with_log<'a>( &'a mut self, undo_log: &'a mut InferCtxtUndoLogs<'tcx>, @@ -104,6 +105,7 @@ impl<'tcx> ProjectionCacheStorage<'tcx> { } impl<'tcx> ProjectionCache<'_, 'tcx> { + #[inline] fn map( &mut self, ) -> SnapshotMapRef<