new solver: make all goal evaluation able to be automatically rerun

compiler/rustc_trait_selection/src/solve/assembly.rs

@@ -224,7 +224,9 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         if goal.predicate.self_ty().is_ty_var() {
             return vec![Candidate {
                 source: CandidateSource::BuiltinImpl,
-                result: self.make_canonical_response(Certainty::AMBIGUOUS).unwrap(),
+                result: self
+                    .evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS)
+                    .unwrap(),
             }];
         }
 
@@ -261,37 +263,26 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         let &ty::Alias(ty::Projection, projection_ty) = goal.predicate.self_ty().kind() else {
             return
         };
-        self.probe(|this| {
-            let normalized_ty = this.next_ty_infer();
+
+        self.probe(|ecx| {
+            let normalized_ty = ecx.next_ty_infer();
             let normalizes_to_goal = goal.with(
                 tcx,
                 ty::Binder::dummy(ty::ProjectionPredicate {
                     projection_ty,
                     term: normalized_ty.into(),
                 }),
             );
-            let normalization_certainty = match this.evaluate_goal(normalizes_to_goal) {
-                Ok((_, certainty)) => certainty,
-                Err(NoSolution) => return,
-            };
-            let normalized_ty = this.resolve_vars_if_possible(normalized_ty);
-
-            // NOTE: Alternatively we could call `evaluate_goal` here and only have a `Normalized` candidate.
-            // This doesn't work as long as we use `CandidateSource` in winnowing.
-            let goal = goal.with(tcx, goal.predicate.with_self_ty(tcx, normalized_ty));
-            let normalized_candidates = this.assemble_and_evaluate_candidates(goal);
-            for mut normalized_candidate in normalized_candidates {
-                normalized_candidate.result =
-                    normalized_candidate.result.unchecked_map(|mut response| {
-                        // FIXME: This currently hides overflow in the normalization step of the self type
-                        // which is probably wrong. Maybe `unify_and` should actually keep overflow as
-                        // we treat it as non-fatal anyways.
-                        response.certainty = response.certainty.unify_and(normalization_certainty);
-                        response
-                    });
-                candidates.push(normalized_candidate);
+            ecx.add_goal(normalizes_to_goal);
+            if let Ok(_) = ecx.try_evaluate_added_goals() {
+                let normalized_ty = ecx.resolve_vars_if_possible(normalized_ty);
+
+                // NOTE: Alternatively we could call `evaluate_goal` here and only have a `Normalized` candidate.
+                // This doesn't work as long as we use `CandidateSource` in winnowing.
+                let goal = goal.with(tcx, goal.predicate.with_self_ty(tcx, normalized_ty));
+                candidates.extend(ecx.assemble_and_evaluate_candidates(goal));
             }
-        })
+        });
     }
 
     fn assemble_impl_candidates<G: GoalKind<'tcx>>(
@@ -516,7 +507,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
                 } else {
                     Certainty::AMBIGUOUS
                 };
-                return self.make_canonical_response(certainty);
+                return self.evaluate_added_goals_and_make_canonical_response(certainty);
             }
         }
 
@@ -538,14 +529,16 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         }
     }
 
-    fn discard_reservation_impl(&self, mut candidate: Candidate<'tcx>) -> Candidate<'tcx> {
+    fn discard_reservation_impl(&mut self, mut candidate: Candidate<'tcx>) -> Candidate<'tcx> {
         if let CandidateSource::Impl(def_id) = candidate.source {
             if let ty::ImplPolarity::Reservation = self.tcx().impl_polarity(def_id) {
                 debug!("Selected reservation impl");
                 // We assemble all candidates inside of a probe so by
                 // making a new canonical response here our result will
                 // have no constraints.
-                candidate.result = self.make_canonical_response(Certainty::AMBIGUOUS).unwrap();
+                candidate.result = self
+                    .evaluate_added_goals_and_make_canonical_response(Certainty::AMBIGUOUS)
+                    .unwrap();
             }
         }
 

compiler/rustc_trait_selection/src/solve/canonical/mod.rs

@@ -48,7 +48,20 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
     /// - `external_constraints`: additional constraints which aren't expressable
     ///   using simple unification of inference variables.
     #[instrument(level = "debug", skip(self))]
-    pub(super) fn make_canonical_response(&self, certainty: Certainty) -> QueryResult<'tcx> {
+    pub(super) fn evaluate_added_goals_and_make_canonical_response(
+        &mut self,
+        certainty: Certainty,
+    ) -> QueryResult<'tcx> {
+        let goals_certainty = self.try_evaluate_added_goals()?;
+        let certainty = certainty.unify_and(goals_certainty);
+
+        if let Certainty::Yes = certainty {
+            assert!(
+                self.nested_goals.is_empty(),
+                "Cannot be certain of query response if unevaluated goals exist"
+            );
+        }
+
         let external_constraints = self.compute_external_query_constraints()?;
 
         let response = Response { var_values: self.var_values, external_constraints, certainty };
@@ -209,7 +222,7 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
             // FIXME: To deal with #105787 I also expect us to emit nested obligations here at
             // some point. We can figure out how to deal with this once we actually have
             // an ICE.
-            let nested_goals = self.eq(param_env, orig, response)?;
+            let nested_goals = self.eq_and_get_goals(param_env, orig, response)?;
             assert!(nested_goals.is_empty(), "{nested_goals:?}");
         }
 

compiler/rustc_trait_selection/src/solve/eval_ctxt.rs

@@ -13,8 +13,7 @@ use rustc_middle::ty::{
 use rustc_span::DUMMY_SP;
 use std::ops::ControlFlow;
 
-use super::search_graph::SearchGraph;
-use super::Goal;
+use super::{search_graph::SearchGraph, Goal};
 
 pub struct EvalCtxt<'a, 'tcx> {
     // FIXME: should be private.
@@ -33,14 +32,41 @@ pub struct EvalCtxt<'a, 'tcx> {
 
     pub(super) search_graph: &'a mut SearchGraph<'tcx>,
 
-    /// This field is used by a debug assertion in [`EvalCtxt::evaluate_goal`],
-    /// see the comment in that method for more details.
-    pub in_projection_eq_hack: bool,
+    pub(super) nested_goals: NestedGoals<'tcx>,
+}
+
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
+pub(super) enum IsNormalizesToHack {
+    Yes,
+    No,
+}
+
+#[derive(Debug, Clone)]
+pub(super) struct NestedGoals<'tcx> {
+    pub(super) normalizes_to_hack_goal: Option<Goal<'tcx, ty::ProjectionPredicate<'tcx>>>,
+    pub(super) goals: Vec<Goal<'tcx, ty::Predicate<'tcx>>>,
+}
+
+impl NestedGoals<'_> {
+    pub(super) fn new() -> Self {
+        Self { normalizes_to_hack_goal: None, goals: Vec::new() }
+    }
+
+    pub(super) fn is_empty(&self) -> bool {
+        self.normalizes_to_hack_goal.is_none() && self.goals.is_empty()
+    }
 }
 
 impl<'tcx> EvalCtxt<'_, 'tcx> {
     pub(super) fn probe<T>(&mut self, f: impl FnOnce(&mut EvalCtxt<'_, 'tcx>) -> T) -> T {
-        self.infcx.probe(|_| f(self))
+        let mut ecx = EvalCtxt {
+            infcx: self.infcx,
+            var_values: self.var_values,
+            max_input_universe: self.max_input_universe,
+            search_graph: self.search_graph,
+            nested_goals: self.nested_goals.clone(),
+        };
+        self.infcx.probe(|_| f(&mut ecx))
     }
 
     pub(super) fn tcx(&self) -> TyCtxt<'tcx> {
@@ -61,6 +87,15 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
         )
     }
 
+    /// Returns a ty infer or a const infer depending on whether `kind` is a `Ty` or `Const`.
+    /// If `kind` is an integer inference variable this will still return a ty infer var.
+    pub(super) fn next_term_infer_of_kind(&self, kind: ty::Term<'tcx>) -> ty::Term<'tcx> {
+        match kind.unpack() {
+            ty::TermKind::Ty(_) => self.next_ty_infer().into(),
+            ty::TermKind::Const(ct) => self.next_const_infer(ct.ty()).into(),
+        }
+    }
+
     /// Is the projection predicate is of the form `exists<T> <Ty as Trait>::Assoc = T`.
     ///
     /// This is the case if the `term` is an inference variable in the innermost universe
@@ -137,6 +172,30 @@ impl<'tcx> EvalCtxt<'_, 'tcx> {
 
     #[instrument(level = "debug", skip(self, param_env), ret)]
     pub(super) fn eq<T: ToTrace<'tcx>>(
+        &mut self,
+        param_env: ty::ParamEnv<'tcx>,
+        lhs: T,
+        rhs: T,
+    ) -> Result<(), NoSolution> {
+        self.infcx
+            .at(&ObligationCause::dummy(), param_env)
+            .eq(DefineOpaqueTypes::No, lhs, rhs)
+            .map(|InferOk { value: (), obligations }| {
+                self.add_goals(obligations.into_iter().map(|o| o.into()));
+            })
+            .map_err(|e| {
+                debug!(?e, "failed to equate");
+                NoSolution
+            })
+    }
+
+    /// Equates two values returning the nested goals without adding them
+    /// to the nested goals of the `EvalCtxt`.
+    ///
+    /// If possible, try using `eq` instead which automatically handles nested
+    /// goals correctly.
+    #[instrument(level = "debug", skip(self, param_env), ret)]
+    pub(super) fn eq_and_get_goals<T: ToTrace<'tcx>>(
         &self,
         param_env: ty::ParamEnv<'tcx>,
         lhs: T,