From 8252ad02c42646fe4512ff6e48b119f1981a3fff Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Tue, 17 Jan 2023 21:00:07 +0000 Subject: [PATCH 01/16] Extract cost checker from inliner. --- .../rustc_mir_transform/src/cost_checker.rs | 94 +++++++++++++++++++ compiler/rustc_mir_transform/src/inline.rs | 91 +----------------- compiler/rustc_mir_transform/src/lib.rs | 1 + 3 files changed, 98 insertions(+), 88 deletions(-) create mode 100644 compiler/rustc_mir_transform/src/cost_checker.rs diff --git a/compiler/rustc_mir_transform/src/cost_checker.rs b/compiler/rustc_mir_transform/src/cost_checker.rs new file mode 100644 index 0000000000000..4c985a1f874e8 --- /dev/null +++ b/compiler/rustc_mir_transform/src/cost_checker.rs @@ -0,0 +1,94 @@ +use rustc_middle::mir::visit::*; +use rustc_middle::mir::*; +use rustc_middle::ty::{self, ParamEnv, TyCtxt}; + +const INSTR_COST: usize = 5; +const CALL_PENALTY: usize = 25; +const LANDINGPAD_PENALTY: usize = 50; +const RESUME_PENALTY: usize = 45; + +/// Verify that the callee body is compatible with the caller. +#[derive(Clone)] +pub(crate) struct CostChecker<'b, 'tcx> { + tcx: TyCtxt<'tcx>, + param_env: ParamEnv<'tcx>, + cost: usize, + callee_body: &'b Body<'tcx>, + instance: ty::Instance<'tcx>, +} + +impl<'b, 'tcx> CostChecker<'b, 'tcx> { + pub fn new( + tcx: TyCtxt<'tcx>, + param_env: ParamEnv<'tcx>, + instance: ty::Instance<'tcx>, + callee_body: &'b Body<'tcx>, + ) -> CostChecker<'b, 'tcx> { + CostChecker { tcx, param_env, callee_body, instance, cost: 0 } + } + + pub fn cost(&self) -> usize { + self.cost + } +} + +impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { + fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) { + // Don't count StorageLive/StorageDead in the inlining cost. + match statement.kind { + StatementKind::StorageLive(_) + | StatementKind::StorageDead(_) + | StatementKind::Deinit(_) + | StatementKind::Nop => {} + _ => self.cost += INSTR_COST, + } + } + + fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, _: Location) { + let tcx = self.tcx; + match terminator.kind { + TerminatorKind::Drop { ref place, unwind, .. } => { + // If the place doesn't actually need dropping, treat it like a regular goto. + let ty = self.instance.instantiate_mir( + tcx, + ty::EarlyBinder::bind(&place.ty(self.callee_body, tcx).ty), + ); + if ty.needs_drop(tcx, self.param_env) { + self.cost += CALL_PENALTY; + if let UnwindAction::Cleanup(_) = unwind { + self.cost += LANDINGPAD_PENALTY; + } + } else { + self.cost += INSTR_COST; + } + } + TerminatorKind::Call { func: Operand::Constant(ref f), unwind, .. } => { + let fn_ty = + self.instance.instantiate_mir(tcx, ty::EarlyBinder::bind(&f.const_.ty())); + self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind() && tcx.is_intrinsic(def_id) { + // Don't give intrinsics the extra penalty for calls + INSTR_COST + } else { + CALL_PENALTY + }; + if let UnwindAction::Cleanup(_) = unwind { + self.cost += LANDINGPAD_PENALTY; + } + } + TerminatorKind::Assert { unwind, .. } => { + self.cost += CALL_PENALTY; + if let UnwindAction::Cleanup(_) = unwind { + self.cost += LANDINGPAD_PENALTY; + } + } + TerminatorKind::UnwindResume => self.cost += RESUME_PENALTY, + TerminatorKind::InlineAsm { unwind, .. } => { + self.cost += INSTR_COST; + if let UnwindAction::Cleanup(_) = unwind { + self.cost += LANDINGPAD_PENALTY; + } + } + _ => self.cost += INSTR_COST, + } + } +} diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 757b2aeca7b50..67cf39b7f837e 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -14,6 +14,7 @@ use rustc_session::config::OptLevel; use rustc_target::abi::FieldIdx; use rustc_target::spec::abi::Abi; +use crate::cost_checker::CostChecker; use crate::simplify::{remove_dead_blocks, CfgSimplifier}; use crate::util; use crate::MirPass; @@ -22,11 +23,6 @@ use std::ops::{Range, RangeFrom}; pub(crate) mod cycle; -const INSTR_COST: usize = 5; -const CALL_PENALTY: usize = 25; -const LANDINGPAD_PENALTY: usize = 50; -const RESUME_PENALTY: usize = 45; - const TOP_DOWN_DEPTH_LIMIT: usize = 5; pub struct Inline; @@ -479,13 +475,7 @@ impl<'tcx> Inliner<'tcx> { // FIXME: Give a bonus to functions with only a single caller - let mut checker = CostChecker { - tcx: self.tcx, - param_env: self.param_env, - instance: callsite.callee, - callee_body, - cost: 0, - }; + let mut checker = CostChecker::new(self.tcx, self.param_env, callsite.callee, callee_body); // Traverse the MIR manually so we can account for the effects of inlining on the CFG. let mut work_list = vec![START_BLOCK]; @@ -530,7 +520,7 @@ impl<'tcx> Inliner<'tcx> { // That attribute is often applied to very large functions that exceed LLVM's (very // generous) inlining threshold. Such functions are very poor MIR inlining candidates. // Always inlining #[inline(always)] functions in MIR, on net, slows down the compiler. - let cost = checker.cost; + let cost = checker.cost(); if cost <= threshold { debug!("INLINING {:?} [cost={} <= threshold={}]", callsite, cost, threshold); Ok(()) @@ -803,81 +793,6 @@ impl<'tcx> Inliner<'tcx> { } } -/// Verify that the callee body is compatible with the caller. -/// -/// This visitor mostly computes the inlining cost, -/// but also needs to verify that types match because of normalization failure. -struct CostChecker<'b, 'tcx> { - tcx: TyCtxt<'tcx>, - param_env: ParamEnv<'tcx>, - cost: usize, - callee_body: &'b Body<'tcx>, - instance: ty::Instance<'tcx>, -} - -impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { - fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) { - // Don't count StorageLive/StorageDead in the inlining cost. - match statement.kind { - StatementKind::StorageLive(_) - | StatementKind::StorageDead(_) - | StatementKind::Deinit(_) - | StatementKind::Nop => {} - _ => self.cost += INSTR_COST, - } - } - - fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, _: Location) { - let tcx = self.tcx; - match terminator.kind { - TerminatorKind::Drop { ref place, unwind, .. } => { - // If the place doesn't actually need dropping, treat it like a regular goto. - let ty = self.instance.instantiate_mir( - tcx, - ty::EarlyBinder::bind(&place.ty(self.callee_body, tcx).ty), - ); - if ty.needs_drop(tcx, self.param_env) { - self.cost += CALL_PENALTY; - if let UnwindAction::Cleanup(_) = unwind { - self.cost += LANDINGPAD_PENALTY; - } - } else { - self.cost += INSTR_COST; - } - } - TerminatorKind::Call { func: Operand::Constant(ref f), unwind, .. } => { - let fn_ty = - self.instance.instantiate_mir(tcx, ty::EarlyBinder::bind(&f.const_.ty())); - self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind() - && tcx.is_intrinsic(def_id) - { - // Don't give intrinsics the extra penalty for calls - INSTR_COST - } else { - CALL_PENALTY - }; - if let UnwindAction::Cleanup(_) = unwind { - self.cost += LANDINGPAD_PENALTY; - } - } - TerminatorKind::Assert { unwind, .. } => { - self.cost += CALL_PENALTY; - if let UnwindAction::Cleanup(_) = unwind { - self.cost += LANDINGPAD_PENALTY; - } - } - TerminatorKind::UnwindResume => self.cost += RESUME_PENALTY, - TerminatorKind::InlineAsm { unwind, .. } => { - self.cost += INSTR_COST; - if let UnwindAction::Cleanup(_) = unwind { - self.cost += LANDINGPAD_PENALTY; - } - } - _ => self.cost += INSTR_COST, - } - } -} - /** * Integrator. * diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 6c0aa51795bdb..c6454a409360a 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -62,6 +62,7 @@ mod const_prop; mod const_prop_lint; mod copy_prop; mod coroutine; +mod cost_checker; mod coverage; mod cross_crate_inline; mod ctfe_limit; From 3cb0c2e3851c8ec129216f63fdf322276dd1eb96 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Fri, 3 Feb 2023 16:47:11 +0000 Subject: [PATCH 02/16] Make instance an option in CostChecker. --- .../rustc_mir_transform/src/cost_checker.rs | 22 +++++++++++-------- compiler/rustc_mir_transform/src/inline.rs | 3 ++- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/compiler/rustc_mir_transform/src/cost_checker.rs b/compiler/rustc_mir_transform/src/cost_checker.rs index 4c985a1f874e8..9bb26693cb244 100644 --- a/compiler/rustc_mir_transform/src/cost_checker.rs +++ b/compiler/rustc_mir_transform/src/cost_checker.rs @@ -1,6 +1,6 @@ use rustc_middle::mir::visit::*; use rustc_middle::mir::*; -use rustc_middle::ty::{self, ParamEnv, TyCtxt}; +use rustc_middle::ty::{self, ParamEnv, Ty, TyCtxt}; const INSTR_COST: usize = 5; const CALL_PENALTY: usize = 25; @@ -14,14 +14,14 @@ pub(crate) struct CostChecker<'b, 'tcx> { param_env: ParamEnv<'tcx>, cost: usize, callee_body: &'b Body<'tcx>, - instance: ty::Instance<'tcx>, + instance: Option>, } impl<'b, 'tcx> CostChecker<'b, 'tcx> { pub fn new( tcx: TyCtxt<'tcx>, param_env: ParamEnv<'tcx>, - instance: ty::Instance<'tcx>, + instance: Option>, callee_body: &'b Body<'tcx>, ) -> CostChecker<'b, 'tcx> { CostChecker { tcx, param_env, callee_body, instance, cost: 0 } @@ -30,6 +30,14 @@ impl<'b, 'tcx> CostChecker<'b, 'tcx> { pub fn cost(&self) -> usize { self.cost } + + fn instantiate_ty(&self, v: Ty<'tcx>) -> Ty<'tcx> { + if let Some(instance) = self.instance { + instance.instantiate_mir(self.tcx, ty::EarlyBinder::bind(&v)) + } else { + v + } + } } impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { @@ -49,10 +57,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { match terminator.kind { TerminatorKind::Drop { ref place, unwind, .. } => { // If the place doesn't actually need dropping, treat it like a regular goto. - let ty = self.instance.instantiate_mir( - tcx, - ty::EarlyBinder::bind(&place.ty(self.callee_body, tcx).ty), - ); + let ty = self.instantiate_ty(place.ty(self.callee_body, tcx).ty); if ty.needs_drop(tcx, self.param_env) { self.cost += CALL_PENALTY; if let UnwindAction::Cleanup(_) = unwind { @@ -63,8 +68,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { } } TerminatorKind::Call { func: Operand::Constant(ref f), unwind, .. } => { - let fn_ty = - self.instance.instantiate_mir(tcx, ty::EarlyBinder::bind(&f.const_.ty())); + let fn_ty = self.instantiate_ty(f.const_.ty()); self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind() && tcx.is_intrinsic(def_id) { // Don't give intrinsics the extra penalty for calls INSTR_COST diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 67cf39b7f837e..8b33e00c63cca 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -475,7 +475,8 @@ impl<'tcx> Inliner<'tcx> { // FIXME: Give a bonus to functions with only a single caller - let mut checker = CostChecker::new(self.tcx, self.param_env, callsite.callee, callee_body); + let mut checker = + CostChecker::new(self.tcx, self.param_env, Some(callsite.callee), callee_body); // Traverse the MIR manually so we can account for the effects of inlining on the CFG. let mut work_list = vec![START_BLOCK]; From 751a079413a920ab380d63cdffbe99cf2476fe89 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Mon, 16 Jan 2023 22:12:36 +0000 Subject: [PATCH 03/16] Implement JumpThreading pass. --- Cargo.lock | 1 + compiler/rustc_middle/src/mir/terminator.rs | 9 + .../rustc_mir_dataflow/src/value_analysis.rs | 107 ++- compiler/rustc_mir_transform/Cargo.toml | 1 + .../rustc_mir_transform/src/jump_threading.rs | 638 ++++++++++++++++++ compiler/rustc_mir_transform/src/lib.rs | 2 + ...ustom_discr.JumpThreading.panic-abort.diff | 57 ++ ...stom_discr.JumpThreading.panic-unwind.diff | 57 ++ ...reading.dfa.JumpThreading.panic-abort.diff | 87 +++ ...eading.dfa.JumpThreading.panic-unwind.diff | 87 +++ ...ppearing_bb.JumpThreading.panic-abort.diff | 57 ++ ...pearing_bb.JumpThreading.panic-unwind.diff | 57 ++ ...icate_chain.JumpThreading.panic-abort.diff | 45 ++ ...cate_chain.JumpThreading.panic-unwind.diff | 45 ++ ...ng.identity.JumpThreading.panic-abort.diff | 139 ++++ ...g.identity.JumpThreading.panic-unwind.diff | 139 ++++ ...tiple_match.JumpThreading.panic-abort.diff | 54 ++ ...iple_match.JumpThreading.panic-unwind.diff | 54 ++ ...mutable_ref.JumpThreading.panic-abort.diff | 56 ++ ...utable_ref.JumpThreading.panic-unwind.diff | 56 ++ ...iscriminant.JumpThreading.panic-abort.diff | 26 + ...scriminant.JumpThreading.panic-unwind.diff | 26 + ...numbered_bb.JumpThreading.panic-abort.diff | 53 ++ ...umbered_bb.JumpThreading.panic-unwind.diff | 53 ++ tests/mir-opt/jump_threading.rs | 291 ++++++++ ...too_complex.JumpThreading.panic-abort.diff | 98 +++ ...oo_complex.JumpThreading.panic-unwind.diff | 98 +++ 27 files changed, 2375 insertions(+), 18 deletions(-) create mode 100644 compiler/rustc_mir_transform/src/jump_threading.rs create mode 100644 tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-abort.diff create mode 100644 tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-unwind.diff create mode 100644 tests/mir-opt/jump_threading.dfa.JumpThreading.panic-abort.diff create mode 100644 tests/mir-opt/jump_threading.dfa.JumpThreading.panic-unwind.diff create mode 100644 tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff create mode 100644 tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff create mode 100644 tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-abort.diff create mode 100644 tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-unwind.diff create mode 100644 tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff create mode 100644 tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff create mode 100644 tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-abort.diff create mode 100644 tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-unwind.diff create mode 100644 tests/mir-opt/jump_threading.mutable_ref.JumpThreading.panic-abort.diff create mode 100644 tests/mir-opt/jump_threading.mutable_ref.JumpThreading.panic-unwind.diff create mode 100644 tests/mir-opt/jump_threading.mutate_discriminant.JumpThreading.panic-abort.diff create mode 100644 tests/mir-opt/jump_threading.mutate_discriminant.JumpThreading.panic-unwind.diff create mode 100644 tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff create mode 100644 tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff create mode 100644 tests/mir-opt/jump_threading.rs create mode 100644 tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-abort.diff create mode 100644 tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-unwind.diff diff --git a/Cargo.lock b/Cargo.lock index a1a02c65d9483..ac5c164e50932 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4242,6 +4242,7 @@ dependencies = [ "coverage_test_macros", "either", "itertools", + "rustc_arena", "rustc_ast", "rustc_attr", "rustc_const_eval", diff --git a/compiler/rustc_middle/src/mir/terminator.rs b/compiler/rustc_middle/src/mir/terminator.rs index e3d346c0698d8..25879e861ed29 100644 --- a/compiler/rustc_middle/src/mir/terminator.rs +++ b/compiler/rustc_middle/src/mir/terminator.rs @@ -28,6 +28,15 @@ impl SwitchTargets { Self { values: smallvec![value], targets: smallvec![then, else_] } } + /// Inverse of `SwitchTargets::static_if`. + pub fn as_static_if(&self) -> Option<(u128, BasicBlock, BasicBlock)> { + if let &[value] = &self.values[..] && let &[then, else_] = &self.targets[..] { + Some((value, then, else_)) + } else { + None + } + } + /// Returns the fallback target that is jumped to when none of the values match the operand. pub fn otherwise(&self) -> BasicBlock { *self.targets.last().unwrap() diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs index 04108aeedf625..77c9a22df79ce 100644 --- a/compiler/rustc_mir_dataflow/src/value_analysis.rs +++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs @@ -463,7 +463,19 @@ impl Clone for State { } } -impl State { +impl State { + pub fn new(init: V, map: &Map) -> State { + let values = IndexVec::from_elem_n(init, map.value_count); + State(StateData::Reachable(values)) + } + + pub fn all(&self, f: impl Fn(&V) -> bool) -> bool { + match self.0 { + StateData::Unreachable => true, + StateData::Reachable(ref values) => values.iter().all(f), + } + } + pub fn is_reachable(&self) -> bool { matches!(&self.0, StateData::Reachable(_)) } @@ -472,7 +484,10 @@ impl State { self.0 = StateData::Unreachable; } - pub fn flood_all(&mut self) { + pub fn flood_all(&mut self) + where + V: HasTop, + { self.flood_all_with(V::TOP) } @@ -482,27 +497,40 @@ impl State { } pub fn flood_with(&mut self, place: PlaceRef<'_>, map: &Map, value: V) { - let StateData::Reachable(values) = &mut self.0 else { return }; - map.for_each_aliasing_place(place, None, &mut |vi| { - values[vi] = value.clone(); - }); + self.flood_with_extra(place, None, map, value) } - pub fn flood(&mut self, place: PlaceRef<'_>, map: &Map) { + pub fn flood(&mut self, place: PlaceRef<'_>, map: &Map) + where + V: HasTop, + { self.flood_with(place, map, V::TOP) } pub fn flood_discr_with(&mut self, place: PlaceRef<'_>, map: &Map, value: V) { - let StateData::Reachable(values) = &mut self.0 else { return }; - map.for_each_aliasing_place(place, Some(TrackElem::Discriminant), &mut |vi| { - values[vi] = value.clone(); - }); + self.flood_with_extra(place, Some(TrackElem::Discriminant), map, value) } - pub fn flood_discr(&mut self, place: PlaceRef<'_>, map: &Map) { + pub fn flood_discr(&mut self, place: PlaceRef<'_>, map: &Map) + where + V: HasTop, + { self.flood_discr_with(place, map, V::TOP) } + pub fn flood_with_extra( + &mut self, + place: PlaceRef<'_>, + tail_elem: Option, + map: &Map, + value: V, + ) { + let StateData::Reachable(values) = &mut self.0 else { return }; + map.for_each_aliasing_place(place, tail_elem, &mut |vi| { + values[vi] = value.clone(); + }); + } + /// Low-level method that assigns to a place. /// This does nothing if the place is not tracked. /// @@ -553,7 +581,10 @@ impl State { } /// Helper method to interpret `target = result`. - pub fn assign(&mut self, target: PlaceRef<'_>, result: ValueOrPlace, map: &Map) { + pub fn assign(&mut self, target: PlaceRef<'_>, result: ValueOrPlace, map: &Map) + where + V: HasTop, + { self.flood(target, map); if let Some(target) = map.find(target) { self.insert_idx(target, result, map); @@ -561,7 +592,10 @@ impl State { } /// Helper method for assignments to a discriminant. - pub fn assign_discr(&mut self, target: PlaceRef<'_>, result: ValueOrPlace, map: &Map) { + pub fn assign_discr(&mut self, target: PlaceRef<'_>, result: ValueOrPlace, map: &Map) + where + V: HasTop, + { self.flood_discr(target, map); if let Some(target) = map.find_discr(target) { self.insert_idx(target, result, map); @@ -569,12 +603,43 @@ impl State { } /// Retrieve the value stored for a place, or ⊤ if it is not tracked. - pub fn get(&self, place: PlaceRef<'_>, map: &Map) -> V { + pub fn try_get(&self, place: PlaceRef<'_>, map: &Map) -> Option { + let place = map.find(place)?; + self.try_get_idx(place, map) + } + + /// Retrieve the value stored for a place, or ⊤ if it is not tracked. + pub fn try_get_discr(&self, place: PlaceRef<'_>, map: &Map) -> Option { + let place = map.find_discr(place)?; + self.try_get_idx(place, map) + } + + /// Retrieve the value stored for a place index, or ⊤ if it is not tracked. + pub fn try_get_idx(&self, place: PlaceIndex, map: &Map) -> Option { + match &self.0 { + StateData::Reachable(values) => { + map.places[place].value_index.map(|v| values[v].clone()) + } + StateData::Unreachable => { + // Because this is unreachable, we can return any value we want. + None + } + } + } + + /// Retrieve the value stored for a place, or ⊤ if it is not tracked. + pub fn get(&self, place: PlaceRef<'_>, map: &Map) -> V + where + V: HasBottom + HasTop, + { map.find(place).map(|place| self.get_idx(place, map)).unwrap_or(V::TOP) } /// Retrieve the value stored for a place, or ⊤ if it is not tracked. - pub fn get_discr(&self, place: PlaceRef<'_>, map: &Map) -> V { + pub fn get_discr(&self, place: PlaceRef<'_>, map: &Map) -> V + where + V: HasBottom + HasTop, + { match map.find_discr(place) { Some(place) => self.get_idx(place, map), None => V::TOP, @@ -582,7 +647,10 @@ impl State { } /// Retrieve the value stored for a place, or ⊤ if it is not tracked. - pub fn get_len(&self, place: PlaceRef<'_>, map: &Map) -> V { + pub fn get_len(&self, place: PlaceRef<'_>, map: &Map) -> V + where + V: HasBottom + HasTop, + { match map.find_len(place) { Some(place) => self.get_idx(place, map), None => V::TOP, @@ -590,7 +658,10 @@ impl State { } /// Retrieve the value stored for a place index, or ⊤ if it is not tracked. - pub fn get_idx(&self, place: PlaceIndex, map: &Map) -> V { + pub fn get_idx(&self, place: PlaceIndex, map: &Map) -> V + where + V: HasBottom + HasTop, + { match &self.0 { StateData::Reachable(values) => { map.places[place].value_index.map(|v| values[v].clone()).unwrap_or(V::TOP) diff --git a/compiler/rustc_mir_transform/Cargo.toml b/compiler/rustc_mir_transform/Cargo.toml index f1198d9bfd375..9ec0bb4ab94ad 100644 --- a/compiler/rustc_mir_transform/Cargo.toml +++ b/compiler/rustc_mir_transform/Cargo.toml @@ -11,6 +11,7 @@ smallvec = { version = "1.8.1", features = ["union", "may_dangle"] } tracing = "0.1" either = "1" rustc_ast = { path = "../rustc_ast" } +rustc_arena = { path = "../rustc_arena" } rustc_attr = { path = "../rustc_attr" } rustc_data_structures = { path = "../rustc_data_structures" } rustc_errors = { path = "../rustc_errors" } diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs new file mode 100644 index 0000000000000..1831184aedae5 --- /dev/null +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -0,0 +1,638 @@ +//! A jump threading optimization. +//! +//! This optimization seeks to replace join-then-switch control flow patterns by straight jumps +//! X = 0 X = 0 +//! ------------\ /-------- ------------ +//! X = 1 X----X SwitchInt(X) => X = 1 +//! ------------/ \-------- ------------ +//! +//! +//! We proceed by walking the cfg backwards starting from each `SwitchInt` terminator, +//! looking for assignments that will turn the `SwitchInt` into a simple `Goto`. +//! +//! The algorithm maintains a set of replacement conditions: +//! - `conditions[place]` contains `Condition { value, polarity: true, target }` +//! if assigning `value` to `place` turns the `SwitchInt` into `Goto { target }`. +//! - `conditions[place]` contains `Condition { value, polarity: false, target }` +//! if assigning anything different from `value` to `place` turns the `SwitchInt` +//! into `Goto { target }`. +//! +//! We then walk the CFG backwards transforming the set of conditions. +//! When we find a fulfilling assignment, we record a `ThreadingOpportunity`. +//! All `ThreadingOpportunity`s are applied to the body, by duplicating blocks if required. +//! +//! The optimization search can be very heavy, as it performs a DFS on MIR starting from +//! each `SwitchInt` terminator. To manage the complexity, we: +//! - bound the maximum depth by a constant `MAX_BACKTRACK`; +//! - we only traverse `Goto` terminators. +//! +//! Likewise, applying the optimisation can create a lot of new MIR, so we bound the instruction +//! cost by `MAX_COST`. + +use rustc_arena::DroplessArena; +use rustc_data_structures::fx::FxHashSet; +use rustc_index::IndexVec; +use rustc_middle::mir::visit::Visitor; +use rustc_middle::mir::*; +use rustc_middle::ty::{self, ScalarInt, Ty, TyCtxt}; +use rustc_mir_dataflow::value_analysis::{Map, PlaceIndex, State, TrackElem}; + +use crate::cost_checker::CostChecker; +use crate::MirPass; + +pub struct JumpThreading; + +const MAX_BACKTRACK: usize = 5; +const MAX_COST: usize = 100; +const MAX_PLACES: usize = 100; + +impl<'tcx> MirPass<'tcx> for JumpThreading { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 4 + } + + #[instrument(skip_all level = "debug")] + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let def_id = body.source.def_id(); + debug!(?def_id); + + let param_env = tcx.param_env_reveal_all_normalized(def_id); + let map = Map::new(tcx, body, Some(MAX_PLACES)); + + let arena = DroplessArena::default(); + let mut finder = + TOFinder { tcx, param_env, body, arena: &arena, map: &map, opportunities: Vec::new() }; + + for (bb, bbdata) in body.basic_blocks.iter_enumerated() { + debug!(?bb, term = ?bbdata.terminator()); + if bbdata.is_cleanup { + continue; + } + let Some((discr, targets)) = bbdata.terminator().kind.as_switch() else { continue }; + let Some(discr) = discr.place() else { continue }; + debug!(?discr, ?bb); + + let discr_ty = discr.ty(body, tcx).ty; + let Ok(discr_layout) = tcx.layout_of(param_env.and(discr_ty)) else { continue }; + + let Some(discr) = finder.map.find(discr.as_ref()) else { continue }; + debug!(?discr); + + let cost = CostChecker::new(tcx, param_env, None, body); + + let mut state = State::new(ConditionSet::default(), &finder.map); + + let conds = if let Some((value, then, else_)) = targets.as_static_if() { + let Some(value) = ScalarInt::try_from_uint(value, discr_layout.size) else { + continue; + }; + arena.alloc_from_iter([ + Condition { value, polarity: true, target: then }, + Condition { value, polarity: false, target: else_ }, + ]) + } else { + arena.alloc_from_iter(targets.iter().filter_map(|(value, target)| { + let value = ScalarInt::try_from_uint(value, discr_layout.size)?; + Some(Condition { value, polarity: true, target }) + })) + }; + let conds = ConditionSet(conds); + state.insert_value_idx(discr, conds, &finder.map); + + finder.find_opportunity(bb, state, cost, 0); + } + + let opportunities = finder.opportunities; + debug!(?opportunities); + if opportunities.is_empty() { + return; + } + + OpportunitySet::new(body, opportunities).apply(body); + } +} + +#[derive(Debug)] +struct ThreadingOpportunity { + /// The list of `BasicBlock`s from the one that found the opportunity to the `SwitchInt`. + chain: Vec, + /// The `SwitchInt` will be replaced by `Goto { target }`. + target: BasicBlock, +} + +struct TOFinder<'tcx, 'a> { + tcx: TyCtxt<'tcx>, + param_env: ty::ParamEnv<'tcx>, + body: &'a Body<'tcx>, + map: &'a Map, + /// We use an arena to avoid cloning the slices when cloning `state`. + arena: &'a DroplessArena, + opportunities: Vec, +} + +#[derive(Copy, Clone, Debug)] +struct Condition { + value: ScalarInt, + /// `true` means `==`, `false` means `!=` + polarity: bool, + target: BasicBlock, +} + +impl Condition { + fn matches(&self, value: ScalarInt) -> bool { + (self.value == value) == self.polarity + } + + fn inv(mut self) -> Self { + self.polarity = !self.polarity; + self + } +} + +#[derive(Copy, Clone, Debug, Default)] +struct ConditionSet<'a>(&'a [Condition]); + +impl<'a> ConditionSet<'a> { + fn iter(self) -> impl Iterator + 'a { + self.0.iter().copied() + } + + fn iter_matches(self, value: ScalarInt) -> impl Iterator + 'a { + self.iter().filter(move |c| c.matches(value)) + } + + fn map(self, arena: &'a DroplessArena, f: impl Fn(Condition) -> Condition) -> ConditionSet<'a> { + ConditionSet(arena.alloc_from_iter(self.iter().map(f))) + } +} + +impl<'tcx, 'a> TOFinder<'tcx, 'a> { + fn is_empty(&self, state: &State>) -> bool { + state.all(|cs| cs.0.is_empty()) + } + + /// Recursion entry point to find threading opportunities. + #[instrument(level = "trace", skip(self, cost), ret)] + fn find_opportunity( + &mut self, + bb: BasicBlock, + mut state: State>, + mut cost: CostChecker<'_, 'tcx>, + depth: usize, + ) { + debug!(cost = ?cost.cost()); + for (statement_index, stmt) in + self.body.basic_blocks[bb].statements.iter().enumerate().rev() + { + if self.is_empty(&state) { + return; + } + + cost.visit_statement(stmt, Location { block: bb, statement_index }); + if cost.cost() > MAX_COST { + return; + } + + // Attempt to turn the `current_condition` on `lhs` into a condition on another place. + self.process_statement(bb, stmt, &mut state); + + // When a statement mutates a place, assignments to that place that happen + // above the mutation cannot fulfill a condition. + // _1 = 5 // Whatever happens here, it won't change the result of a `SwitchInt`. + // _1 = 6 + if let Some((lhs, tail)) = self.mutated_statement(stmt) { + state.flood_with_extra(lhs.as_ref(), tail, self.map, ConditionSet::default()); + } + } + + if self.is_empty(&state) || depth >= MAX_BACKTRACK { + return; + } + + let last_non_rec = self.opportunities.len(); + + let predecessors = &self.body.basic_blocks.predecessors()[bb]; + if let &[pred] = &predecessors[..] && bb != START_BLOCK { + match &self.body.basic_blocks[pred].terminator().kind { + TerminatorKind::Goto { .. } => self.find_opportunity(pred, state, cost, depth), + TerminatorKind::SwitchInt { discr, targets } => { + self.process_switch_int(state, discr, targets, bb); + } + _ => {} + } + } else { + for &pred in predecessors { + if matches!( + self.body.basic_blocks[pred].terminator().kind, + TerminatorKind::Goto { .. } + ) { + self.find_opportunity(pred, state.clone(), cost.clone(), depth + 1); + } + } + } + + let new_tos = &mut self.opportunities[last_non_rec..]; + debug!(?new_tos); + + // Try to deduplicate threading opportunities. + if new_tos.len() > 1 + && new_tos.len() == predecessors.len() + && predecessors + .iter() + .zip(new_tos.iter()) + .all(|(&pred, to)| to.chain == &[pred] && to.target == new_tos[0].target) + { + // All predecessors have a threading opportunity, and they all point to the same block. + debug!(?new_tos, "dedup"); + let first = &mut new_tos[0]; + *first = ThreadingOpportunity { chain: vec![bb], target: first.target }; + self.opportunities.truncate(last_non_rec + 1); + return; + } + + for op in self.opportunities[last_non_rec..].iter_mut() { + op.chain.push(bb); + } + } + + /// Extract the mutated place from a statement. + #[instrument(level = "trace", skip(self), ret)] + fn mutated_statement( + &self, + stmt: &Statement<'tcx>, + ) -> Option<(Place<'tcx>, Option)> { + match stmt.kind { + StatementKind::Assign(box (place, _)) + | StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume( + Operand::Copy(place) | Operand::Move(place), + )) + | StatementKind::Deinit(box place) => Some((place, None)), + StatementKind::SetDiscriminant { box place, variant_index: _ } => { + Some((place, Some(TrackElem::Discriminant))) + } + StatementKind::StorageLive(local) | StatementKind::StorageDead(local) => { + Some((Place::from(local), None)) + } + StatementKind::Retag(..) + | StatementKind::Intrinsic(..) + | StatementKind::AscribeUserType(..) + | StatementKind::Coverage(..) + | StatementKind::FakeRead(..) + | StatementKind::ConstEvalCounter + | StatementKind::PlaceMention(..) + | StatementKind::Nop => None, + } + } + + #[instrument(level = "trace", skip(self))] + fn process_operand( + &mut self, + bb: BasicBlock, + lhs: PlaceIndex, + rhs: &Operand<'tcx>, + state: &mut State>, + ) -> Option { + let register_opportunity = |c: Condition| { + debug!(?bb, ?c.target, "register"); + self.opportunities.push(ThreadingOpportunity { chain: vec![bb], target: c.target }) + }; + + match rhs { + // If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`. + Operand::Constant(constant) => { + let conditions = state.try_get_idx(lhs, self.map)?; + let constant = + constant.const_.normalize(self.tcx, self.param_env).try_to_scalar_int()?; + conditions.iter_matches(constant).for_each(register_opportunity); + } + // Transfer the conditions on the copied rhs. + Operand::Move(rhs) | Operand::Copy(rhs) => { + let rhs = self.map.find(rhs.as_ref())?; + state.insert_place_idx(rhs, lhs, self.map); + } + } + + None + } + + #[instrument(level = "trace", skip(self))] + fn process_statement( + &mut self, + bb: BasicBlock, + stmt: &Statement<'tcx>, + state: &mut State>, + ) -> Option { + let register_opportunity = |c: Condition| { + debug!(?bb, ?c.target, "register"); + self.opportunities.push(ThreadingOpportunity { chain: vec![bb], target: c.target }) + }; + + // Below, `lhs` is the return value of `mutated_statement`, + // the place to which `conditions` apply. + + let discriminant_for_variant = |enum_ty: Ty<'tcx>, variant_index| { + let discr = enum_ty.discriminant_for_variant(self.tcx, variant_index)?; + let discr_layout = self.tcx.layout_of(self.param_env.and(discr.ty)).ok()?; + let scalar = ScalarInt::try_from_uint(discr.val, discr_layout.size)?; + Some(Operand::const_from_scalar( + self.tcx, + discr.ty, + scalar.into(), + rustc_span::DUMMY_SP, + )) + }; + + match &stmt.kind { + // If we expect `discriminant(place) ?= A`, + // we have an opportunity if `variant_index ?= A`. + StatementKind::SetDiscriminant { box place, variant_index } => { + let discr_target = self.map.find_discr(place.as_ref())?; + let enum_ty = place.ty(self.body, self.tcx).ty; + let discr = discriminant_for_variant(enum_ty, *variant_index)?; + self.process_operand(bb, discr_target, &discr, state)?; + } + // If we expect `lhs ?= true`, we have an opportunity if we assume `lhs == true`. + StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume( + Operand::Copy(place) | Operand::Move(place), + )) => { + let conditions = state.try_get(place.as_ref(), self.map)?; + conditions.iter_matches(ScalarInt::TRUE).for_each(register_opportunity); + } + StatementKind::Assign(box (lhs_place, rhs)) => { + if let Some(lhs) = self.map.find(lhs_place.as_ref()) { + match rhs { + Rvalue::Use(operand) => self.process_operand(bb, lhs, operand, state)?, + // Transfer the conditions on the copy rhs. + Rvalue::CopyForDeref(rhs) => { + self.process_operand(bb, lhs, &Operand::Copy(*rhs), state)? + } + Rvalue::Discriminant(rhs) => { + let rhs = self.map.find_discr(rhs.as_ref())?; + state.insert_place_idx(rhs, lhs, self.map); + } + // If we expect `lhs ?= A`, we have an opportunity if we assume `constant == A`. + Rvalue::Aggregate(box ref kind, ref operands) => { + let agg_ty = lhs_place.ty(self.body, self.tcx).ty; + let lhs = match kind { + // Do not support unions. + AggregateKind::Adt(.., Some(_)) => return None, + AggregateKind::Adt(_, variant_index, ..) if agg_ty.is_enum() => { + if let Some(discr_target) = self.map.apply(lhs, TrackElem::Discriminant) + && let Some(discr_value) = discriminant_for_variant(agg_ty, *variant_index) + { + self.process_operand(bb, discr_target, &discr_value, state); + } + self.map.apply(lhs, TrackElem::Variant(*variant_index))? + } + _ => lhs, + }; + for (field_index, operand) in operands.iter_enumerated() { + if let Some(field) = + self.map.apply(lhs, TrackElem::Field(field_index)) + { + self.process_operand(bb, field, operand, state); + } + } + } + // Transfer the conditions on the copy rhs, after inversing polarity. + Rvalue::UnaryOp(UnOp::Not, Operand::Move(place) | Operand::Copy(place)) => { + let conditions = state.try_get_idx(lhs, self.map)?; + let place = self.map.find(place.as_ref())?; + let conds = conditions.map(self.arena, Condition::inv); + state.insert_value_idx(place, conds, self.map); + } + // We expect `lhs ?= A`. We found `lhs = Eq(rhs, B)`. + // Create a condition on `rhs ?= B`. + Rvalue::BinaryOp( + op, + box ( + Operand::Move(place) | Operand::Copy(place), + Operand::Constant(value), + ) + | box ( + Operand::Constant(value), + Operand::Move(place) | Operand::Copy(place), + ), + ) => { + let conditions = state.try_get_idx(lhs, self.map)?; + let place = self.map.find(place.as_ref())?; + let equals = match op { + BinOp::Eq => ScalarInt::TRUE, + BinOp::Ne => ScalarInt::FALSE, + _ => return None, + }; + let value = value + .const_ + .normalize(self.tcx, self.param_env) + .try_to_scalar_int()?; + let conds = conditions.map(self.arena, |c| Condition { + value, + polarity: c.matches(equals), + ..c + }); + state.insert_value_idx(place, conds, self.map); + } + + _ => {} + } + } + } + _ => {} + } + + None + } + + #[instrument(level = "trace", skip(self))] + fn process_switch_int( + &mut self, + state: State>, + discr: &Operand<'tcx>, + targets: &SwitchTargets, + bb: BasicBlock, + ) -> Option { + debug_assert_ne!(bb, START_BLOCK); + debug_assert_eq!(self.body.basic_blocks.predecessors()[bb].len(), 1); + + let discr = discr.place()?; + let discr_ty = discr.ty(self.body, self.tcx).ty; + let discr_layout = self.tcx.layout_of(self.param_env.and(discr_ty)).ok()?; + let conditions = state.try_get(discr.as_ref(), self.map)?; + + if let Some((value, _)) = targets.iter().find(|&(_, target)| target == bb) { + let value = ScalarInt::try_from_uint(value, discr_layout.size)?; + debug_assert_eq!(targets.iter().filter(|&(_, target)| target == bb).count(), 1); + + // We are inside `bb`. Since we have a single predecessor, we know we passed + // through the `SwitchInt` before arriving here. Therefore, we know that + // `discr == value`. If one condition can be fulfilled by `discr == value`, + // that's an opportunity. + for c in conditions.iter_matches(value) { + debug!(?bb, ?c.target, "register"); + self.opportunities.push(ThreadingOpportunity { chain: vec![], target: c.target }); + } + } else if bb == targets.otherwise() { + let (value, _, _) = targets.as_static_if()?; + let value = ScalarInt::try_from_uint(value, discr_layout.size)?; + + // Likewise, we know that `discr != value`. That's a must weaker information, + // so we can only match the exact same condition. + for c in conditions.iter() { + if c.value == value && c.polarity == false { + debug!(?bb, ?c.target, "register"); + self.opportunities + .push(ThreadingOpportunity { chain: vec![], target: c.target }); + } + } + } + + None + } +} + +struct OpportunitySet { + opportunities: Vec, + /// For each bb, give the TOs in which it appears. The pair corresponds to the index + /// in `opportunities` and the index in `ThreadingOpportunity::chain`. + involving_tos: IndexVec>, + /// Cache the number of predecessors for each block, as we clear the basic block cache.. + predecessors: IndexVec, +} + +impl OpportunitySet { + fn new(body: &Body<'_>, opportunities: Vec) -> OpportunitySet { + let mut involving_tos = IndexVec::from_elem(Vec::new(), &body.basic_blocks); + for (index, to) in opportunities.iter().enumerate() { + for (ibb, &bb) in to.chain.iter().enumerate() { + involving_tos[bb].push((index, ibb)); + } + involving_tos[to.target].push((index, to.chain.len())); + } + let predecessors = predecessor_count(body); + OpportunitySet { opportunities, involving_tos, predecessors } + } + + /// Apply the opportunities on the graph. + fn apply(&mut self, body: &mut Body<'_>) { + for i in 0..self.opportunities.len() { + self.apply_once(i, body); + } + } + + #[instrument(level = "trace", skip(self, body))] + fn apply_once(&mut self, index: usize, body: &mut Body<'_>) { + debug!(?self.predecessors); + debug!(?self.involving_tos); + + // Check that `predecessors` satisfies its invariant. + debug_assert_eq!(self.predecessors, predecessor_count(body)); + + // Remove the TO from the vector to allow modifying the other ones later. + let op = &mut self.opportunities[index]; + debug!(?op); + let op_chain = std::mem::take(&mut op.chain); + let op_target = op.target; + debug_assert_eq!(op_chain.len(), op_chain.iter().collect::>().len()); + + let Some((current, chain)) = op_chain.split_first() else { return }; + let basic_blocks = body.basic_blocks.as_mut(); + + // Invariant: we never change the meaning of the program. + let mut current = *current; + for &succ in chain { + debug!(?current, ?succ); + + // `succ` must be a successor of `current`. If it is not, this means this TO is not + // satisfiable, so we bail out. + if basic_blocks[current].terminator().successors().find(|s| *s == succ).is_none() { + debug!("impossible"); + return; + } + + // Fast path: `succ` is only used once, so we can reuse it directly. + if self.predecessors[succ] == 1 { + debug!("single"); + current = succ; + continue; + } + + let new_succ = basic_blocks.push(basic_blocks[succ].clone()); + debug!(?new_succ); + + // Replace `succ` by `new_succ` where it appears. + let mut num_edges = 0; + for s in basic_blocks[current].terminator_mut().successors_mut() { + if *s == succ { + *s = new_succ; + num_edges += 1; + } + } + + // Update predecessors with the new block. + let _new_succ = self.predecessors.push(num_edges); + debug_assert_eq!(new_succ, _new_succ); + self.predecessors[succ] -= num_edges; + self.update_predecessor_count(basic_blocks[new_succ].terminator(), Update::Incr); + + // Replace the `current -> succ` edge by `current -> new_succ` in all the following + // TOs. This is necessary to avoid trying to thread through a non-existing edge. We + // use `involving_tos` here to avoid traversing the full set of TOs on each iteration. + let mut new_involved = Vec::new(); + for &(to_index, in_to_index) in &self.involving_tos[current] { + // That TO has already been applied, do nothing. + if to_index <= index { + continue; + } + + let other_to = &mut self.opportunities[to_index]; + if other_to.chain.get(in_to_index) != Some(¤t) { + continue; + } + let s = other_to.chain.get_mut(in_to_index + 1).unwrap_or(&mut other_to.target); + if *s == succ { + // `other_to` references the `current -> succ` edge, so replace `succ`. + *s = new_succ; + new_involved.push((to_index, in_to_index + 1)); + } + } + // Following TOs new reference `new_succ`, so we will need to update them if we + // duplicate `new_succ` later. + let _new_succ = self.involving_tos.push(new_involved); + debug_assert_eq!(new_succ, _new_succ); + + current = new_succ; + } + + let current = &mut basic_blocks[current]; + self.update_predecessor_count(current.terminator(), Update::Decr); + current.terminator_mut().kind = TerminatorKind::Goto { target: op_target }; + self.predecessors[op_target] += 1; + } + + fn update_predecessor_count(&mut self, terminator: &Terminator<'_>, incr: Update) { + match incr { + Update::Incr => { + for s in terminator.successors() { + self.predecessors[s] += 1; + } + } + Update::Decr => { + for s in terminator.successors() { + self.predecessors[s] -= 1; + } + } + } + } +} + +fn predecessor_count(body: &Body<'_>) -> IndexVec { + let mut predecessors: IndexVec<_, _> = + body.basic_blocks.predecessors().iter().map(|ps| ps.len()).collect(); + predecessors[START_BLOCK] += 1; // Account for the implicit entry edge. + predecessors +} + +enum Update { + Incr, + Decr, +} diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index c6454a409360a..9aaa54110bdfe 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -82,6 +82,7 @@ mod function_item_references; mod gvn; pub mod inline; mod instsimplify; +mod jump_threading; mod large_enums; mod lower_intrinsics; mod lower_slice_len; @@ -572,6 +573,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { &dataflow_const_prop::DataflowConstProp, &const_debuginfo::ConstDebugInfo, &o1(simplify_branches::SimplifyConstCondition::AfterConstProp), + &jump_threading::JumpThreading, &early_otherwise_branch::EarlyOtherwiseBranch, &simplify_comparison_integral::SimplifyComparisonIntegral, &dead_store_elimination::DeadStoreElimination, diff --git a/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-abort.diff new file mode 100644 index 0000000000000..462cc20778580 --- /dev/null +++ b/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-abort.diff @@ -0,0 +1,57 @@ +- // MIR for `custom_discr` before JumpThreading ++ // MIR for `custom_discr` after JumpThreading + + fn custom_discr(_1: bool) -> u8 { + debug x => _1; + let mut _0: u8; + let mut _2: CustomDiscr; + let mut _3: bool; + let mut _4: u8; + + bb0: { + StorageLive(_2); + StorageLive(_3); + _3 = _1; + switchInt(move _3) -> [0: bb2, otherwise: bb1]; + } + + bb1: { + _2 = CustomDiscr::A; +- goto -> bb3; ++ goto -> bb7; + } + + bb2: { + _2 = CustomDiscr::B; + goto -> bb3; + } + + bb3: { + StorageDead(_3); + _4 = discriminant(_2); +- switchInt(move _4) -> [35: bb5, otherwise: bb4]; ++ goto -> bb4; + } + + bb4: { + _0 = const 13_u8; + goto -> bb6; + } + + bb5: { + _0 = const 5_u8; + goto -> bb6; + } + + bb6: { + StorageDead(_2); + return; ++ } ++ ++ bb7: { ++ StorageDead(_3); ++ _4 = discriminant(_2); ++ goto -> bb5; + } + } + diff --git a/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-unwind.diff new file mode 100644 index 0000000000000..462cc20778580 --- /dev/null +++ b/tests/mir-opt/jump_threading.custom_discr.JumpThreading.panic-unwind.diff @@ -0,0 +1,57 @@ +- // MIR for `custom_discr` before JumpThreading ++ // MIR for `custom_discr` after JumpThreading + + fn custom_discr(_1: bool) -> u8 { + debug x => _1; + let mut _0: u8; + let mut _2: CustomDiscr; + let mut _3: bool; + let mut _4: u8; + + bb0: { + StorageLive(_2); + StorageLive(_3); + _3 = _1; + switchInt(move _3) -> [0: bb2, otherwise: bb1]; + } + + bb1: { + _2 = CustomDiscr::A; +- goto -> bb3; ++ goto -> bb7; + } + + bb2: { + _2 = CustomDiscr::B; + goto -> bb3; + } + + bb3: { + StorageDead(_3); + _4 = discriminant(_2); +- switchInt(move _4) -> [35: bb5, otherwise: bb4]; ++ goto -> bb4; + } + + bb4: { + _0 = const 13_u8; + goto -> bb6; + } + + bb5: { + _0 = const 5_u8; + goto -> bb6; + } + + bb6: { + StorageDead(_2); + return; ++ } ++ ++ bb7: { ++ StorageDead(_3); ++ _4 = discriminant(_2); ++ goto -> bb5; + } + } + diff --git a/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-abort.diff new file mode 100644 index 0000000000000..bad733268a30e --- /dev/null +++ b/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-abort.diff @@ -0,0 +1,87 @@ +- // MIR for `dfa` before JumpThreading ++ // MIR for `dfa` after JumpThreading + + fn dfa() -> () { + let mut _0: (); + let mut _1: DFA; + let mut _2: !; + let mut _3: (); + let mut _4: isize; + let mut _5: DFA; + let mut _6: DFA; + let mut _7: DFA; + let mut _8: !; + scope 1 { + debug state => _1; + } + + bb0: { + StorageLive(_1); + _1 = DFA::A; + StorageLive(_2); +- goto -> bb1; ++ goto -> bb7; + } + + bb1: { + _4 = discriminant(_1); +- switchInt(move _4) -> [0: bb4, 1: bb5, 2: bb6, 3: bb2, otherwise: bb3]; ++ goto -> bb2; + } + + bb2: { + _0 = const (); + StorageDead(_2); + StorageDead(_1); + return; + } + + bb3: { + unreachable; + } + + bb4: { + StorageLive(_5); + _5 = DFA::B; + _1 = move _5; + _3 = const (); + StorageDead(_5); +- goto -> bb1; ++ goto -> bb8; + } + + bb5: { + StorageLive(_6); + _6 = DFA::C; + _1 = move _6; + _3 = const (); + StorageDead(_6); +- goto -> bb1; ++ goto -> bb9; + } + + bb6: { + StorageLive(_7); + _7 = DFA::D; + _1 = move _7; + _3 = const (); + StorageDead(_7); + goto -> bb1; ++ } ++ ++ bb7: { ++ _4 = discriminant(_1); ++ goto -> bb4; ++ } ++ ++ bb8: { ++ _4 = discriminant(_1); ++ goto -> bb5; ++ } ++ ++ bb9: { ++ _4 = discriminant(_1); ++ goto -> bb6; + } + } + diff --git a/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-unwind.diff new file mode 100644 index 0000000000000..bad733268a30e --- /dev/null +++ b/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-unwind.diff @@ -0,0 +1,87 @@ +- // MIR for `dfa` before JumpThreading ++ // MIR for `dfa` after JumpThreading + + fn dfa() -> () { + let mut _0: (); + let mut _1: DFA; + let mut _2: !; + let mut _3: (); + let mut _4: isize; + let mut _5: DFA; + let mut _6: DFA; + let mut _7: DFA; + let mut _8: !; + scope 1 { + debug state => _1; + } + + bb0: { + StorageLive(_1); + _1 = DFA::A; + StorageLive(_2); +- goto -> bb1; ++ goto -> bb7; + } + + bb1: { + _4 = discriminant(_1); +- switchInt(move _4) -> [0: bb4, 1: bb5, 2: bb6, 3: bb2, otherwise: bb3]; ++ goto -> bb2; + } + + bb2: { + _0 = const (); + StorageDead(_2); + StorageDead(_1); + return; + } + + bb3: { + unreachable; + } + + bb4: { + StorageLive(_5); + _5 = DFA::B; + _1 = move _5; + _3 = const (); + StorageDead(_5); +- goto -> bb1; ++ goto -> bb8; + } + + bb5: { + StorageLive(_6); + _6 = DFA::C; + _1 = move _6; + _3 = const (); + StorageDead(_6); +- goto -> bb1; ++ goto -> bb9; + } + + bb6: { + StorageLive(_7); + _7 = DFA::D; + _1 = move _7; + _3 = const (); + StorageDead(_7); + goto -> bb1; ++ } ++ ++ bb7: { ++ _4 = discriminant(_1); ++ goto -> bb4; ++ } ++ ++ bb8: { ++ _4 = discriminant(_1); ++ goto -> bb5; ++ } ++ ++ bb9: { ++ _4 = discriminant(_1); ++ goto -> bb6; + } + } + diff --git a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff new file mode 100644 index 0000000000000..d3f737b324c93 --- /dev/null +++ b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff @@ -0,0 +1,57 @@ +- // MIR for `disappearing_bb` before JumpThreading ++ // MIR for `disappearing_bb` after JumpThreading + + fn disappearing_bb(_1: u8) -> u8 { + let mut _0: u8; + let mut _2: i8; + let mut _3: bool; + let mut _4: bool; + + bb0: { + _4 = const false; + _3 = const false; + _4 = const true; + _3 = const true; + switchInt(_1) -> [0: bb3, 1: bb3, 2: bb1, otherwise: bb2]; + } + + bb1: { + _3 = const false; +- goto -> bb4; ++ goto -> bb9; + } + + bb2: { + unreachable; + } + + bb3: { + _4 = const false; + goto -> bb4; + } + + bb4: { + switchInt(_3) -> [0: bb5, otherwise: bb7]; + } + + bb5: { + switchInt(_4) -> [0: bb6, otherwise: bb8]; + } + + bb6: { + return; + } + + bb7: { + goto -> bb5; + } + + bb8: { + goto -> bb6; ++ } ++ ++ bb9: { ++ goto -> bb5; + } + } + diff --git a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff new file mode 100644 index 0000000000000..d3f737b324c93 --- /dev/null +++ b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff @@ -0,0 +1,57 @@ +- // MIR for `disappearing_bb` before JumpThreading ++ // MIR for `disappearing_bb` after JumpThreading + + fn disappearing_bb(_1: u8) -> u8 { + let mut _0: u8; + let mut _2: i8; + let mut _3: bool; + let mut _4: bool; + + bb0: { + _4 = const false; + _3 = const false; + _4 = const true; + _3 = const true; + switchInt(_1) -> [0: bb3, 1: bb3, 2: bb1, otherwise: bb2]; + } + + bb1: { + _3 = const false; +- goto -> bb4; ++ goto -> bb9; + } + + bb2: { + unreachable; + } + + bb3: { + _4 = const false; + goto -> bb4; + } + + bb4: { + switchInt(_3) -> [0: bb5, otherwise: bb7]; + } + + bb5: { + switchInt(_4) -> [0: bb6, otherwise: bb8]; + } + + bb6: { + return; + } + + bb7: { + goto -> bb5; + } + + bb8: { + goto -> bb6; ++ } ++ ++ bb9: { ++ goto -> bb5; + } + } + diff --git a/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-abort.diff new file mode 100644 index 0000000000000..adcedfb366700 --- /dev/null +++ b/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-abort.diff @@ -0,0 +1,45 @@ +- // MIR for `duplicate_chain` before JumpThreading ++ // MIR for `duplicate_chain` after JumpThreading + + fn duplicate_chain(_1: bool) -> u8 { + let mut _0: u8; + let mut _2: u8; + let mut _3: i32; + let mut _4: i32; + + bb0: { + switchInt(_1) -> [1: bb1, otherwise: bb2]; + } + + bb1: { + _2 = const 5_u8; + goto -> bb3; + } + + bb2: { + _2 = const 5_u8; + goto -> bb3; + } + + bb3: { + _3 = const 13_i32; + goto -> bb4; + } + + bb4: { + _4 = const 15_i32; +- switchInt(_2) -> [5: bb5, otherwise: bb6]; ++ goto -> bb5; + } + + bb5: { + _0 = const 7_u8; + return; + } + + bb6: { + _0 = const 9_u8; + return; + } + } + diff --git a/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-unwind.diff new file mode 100644 index 0000000000000..adcedfb366700 --- /dev/null +++ b/tests/mir-opt/jump_threading.duplicate_chain.JumpThreading.panic-unwind.diff @@ -0,0 +1,45 @@ +- // MIR for `duplicate_chain` before JumpThreading ++ // MIR for `duplicate_chain` after JumpThreading + + fn duplicate_chain(_1: bool) -> u8 { + let mut _0: u8; + let mut _2: u8; + let mut _3: i32; + let mut _4: i32; + + bb0: { + switchInt(_1) -> [1: bb1, otherwise: bb2]; + } + + bb1: { + _2 = const 5_u8; + goto -> bb3; + } + + bb2: { + _2 = const 5_u8; + goto -> bb3; + } + + bb3: { + _3 = const 13_i32; + goto -> bb4; + } + + bb4: { + _4 = const 15_i32; +- switchInt(_2) -> [5: bb5, otherwise: bb6]; ++ goto -> bb5; + } + + bb5: { + _0 = const 7_u8; + return; + } + + bb6: { + _0 = const 9_u8; + return; + } + } + diff --git a/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff new file mode 100644 index 0000000000000..f17c9ba3f9fc4 --- /dev/null +++ b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-abort.diff @@ -0,0 +1,139 @@ +- // MIR for `identity` before JumpThreading ++ // MIR for `identity` after JumpThreading + + fn identity(_1: Result) -> Result { + debug x => _1; + let mut _0: std::result::Result; + let mut _2: i32; + let mut _3: std::ops::ControlFlow, i32>; + let mut _4: std::result::Result; + let mut _5: isize; + let _6: std::result::Result; + let mut _7: !; + let mut _8: std::result::Result; + let _9: i32; + scope 1 { + debug residual => _6; + scope 2 { + scope 8 (inlined #[track_caller] as FromResidual>>::from_residual) { + debug residual => _8; + let _14: i32; + let mut _15: i32; + scope 9 { + debug e => _14; + scope 10 (inlined >::from) { + debug t => _14; + } + } + } + } + } + scope 3 { + debug val => _9; + scope 4 { + } + } + scope 5 (inlined as Try>::branch) { + debug self => _4; + let mut _10: isize; + let _11: i32; + let _12: i32; + let mut _13: std::result::Result; + scope 6 { + debug v => _11; + } + scope 7 { + debug e => _12; + } + } + + bb0: { + StorageLive(_2); + StorageLive(_3); + StorageLive(_4); + _4 = _1; + StorageLive(_10); + StorageLive(_11); + StorageLive(_12); + _10 = discriminant(_4); + switchInt(move _10) -> [0: bb8, 1: bb6, otherwise: bb7]; + } + + bb1: { + StorageDead(_12); + StorageDead(_11); + StorageDead(_10); + StorageDead(_4); + _5 = discriminant(_3); +- switchInt(move _5) -> [0: bb2, 1: bb4, otherwise: bb3]; ++ goto -> bb2; + } + + bb2: { + StorageLive(_9); + _9 = ((_3 as Continue).0: i32); + _2 = _9; + StorageDead(_9); + _0 = Result::::Ok(move _2); + StorageDead(_2); + StorageDead(_3); + goto -> bb5; + } + + bb3: { + unreachable; + } + + bb4: { + StorageLive(_6); + _6 = ((_3 as Break).0: std::result::Result); + StorageLive(_8); + _8 = _6; + StorageLive(_14); + _14 = move ((_8 as Err).0: i32); + StorageLive(_15); + _15 = move _14; + _0 = Result::::Err(move _15); + StorageDead(_15); + StorageDead(_14); + StorageDead(_8); + StorageDead(_6); + StorageDead(_2); + StorageDead(_3); + goto -> bb5; + } + + bb5: { + return; + } + + bb6: { + _12 = move ((_4 as Err).0: i32); + StorageLive(_13); + _13 = Result::::Err(move _12); + _3 = ControlFlow::, i32>::Break(move _13); + StorageDead(_13); +- goto -> bb1; ++ goto -> bb9; + } + + bb7: { + unreachable; + } + + bb8: { + _11 = move ((_4 as Ok).0: i32); + _3 = ControlFlow::, i32>::Continue(move _11); + goto -> bb1; ++ } ++ ++ bb9: { ++ StorageDead(_12); ++ StorageDead(_11); ++ StorageDead(_10); ++ StorageDead(_4); ++ _5 = discriminant(_3); ++ goto -> bb4; + } + } + diff --git a/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff new file mode 100644 index 0000000000000..f17c9ba3f9fc4 --- /dev/null +++ b/tests/mir-opt/jump_threading.identity.JumpThreading.panic-unwind.diff @@ -0,0 +1,139 @@ +- // MIR for `identity` before JumpThreading ++ // MIR for `identity` after JumpThreading + + fn identity(_1: Result) -> Result { + debug x => _1; + let mut _0: std::result::Result; + let mut _2: i32; + let mut _3: std::ops::ControlFlow, i32>; + let mut _4: std::result::Result; + let mut _5: isize; + let _6: std::result::Result; + let mut _7: !; + let mut _8: std::result::Result; + let _9: i32; + scope 1 { + debug residual => _6; + scope 2 { + scope 8 (inlined #[track_caller] as FromResidual>>::from_residual) { + debug residual => _8; + let _14: i32; + let mut _15: i32; + scope 9 { + debug e => _14; + scope 10 (inlined >::from) { + debug t => _14; + } + } + } + } + } + scope 3 { + debug val => _9; + scope 4 { + } + } + scope 5 (inlined as Try>::branch) { + debug self => _4; + let mut _10: isize; + let _11: i32; + let _12: i32; + let mut _13: std::result::Result; + scope 6 { + debug v => _11; + } + scope 7 { + debug e => _12; + } + } + + bb0: { + StorageLive(_2); + StorageLive(_3); + StorageLive(_4); + _4 = _1; + StorageLive(_10); + StorageLive(_11); + StorageLive(_12); + _10 = discriminant(_4); + switchInt(move _10) -> [0: bb8, 1: bb6, otherwise: bb7]; + } + + bb1: { + StorageDead(_12); + StorageDead(_11); + StorageDead(_10); + StorageDead(_4); + _5 = discriminant(_3); +- switchInt(move _5) -> [0: bb2, 1: bb4, otherwise: bb3]; ++ goto -> bb2; + } + + bb2: { + StorageLive(_9); + _9 = ((_3 as Continue).0: i32); + _2 = _9; + StorageDead(_9); + _0 = Result::::Ok(move _2); + StorageDead(_2); + StorageDead(_3); + goto -> bb5; + } + + bb3: { + unreachable; + } + + bb4: { + StorageLive(_6); + _6 = ((_3 as Break).0: std::result::Result); + StorageLive(_8); + _8 = _6; + StorageLive(_14); + _14 = move ((_8 as Err).0: i32); + StorageLive(_15); + _15 = move _14; + _0 = Result::::Err(move _15); + StorageDead(_15); + StorageDead(_14); + StorageDead(_8); + StorageDead(_6); + StorageDead(_2); + StorageDead(_3); + goto -> bb5; + } + + bb5: { + return; + } + + bb6: { + _12 = move ((_4 as Err).0: i32); + StorageLive(_13); + _13 = Result::::Err(move _12); + _3 = ControlFlow::, i32>::Break(move _13); + StorageDead(_13); +- goto -> bb1; ++ goto -> bb9; + } + + bb7: { + unreachable; + } + + bb8: { + _11 = move ((_4 as Ok).0: i32); + _3 = ControlFlow::, i32>::Continue(move _11); + goto -> bb1; ++ } ++ ++ bb9: { ++ StorageDead(_12); ++ StorageDead(_11); ++ StorageDead(_10); ++ StorageDead(_4); ++ _5 = discriminant(_3); ++ goto -> bb4; + } + } + diff --git a/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-abort.diff new file mode 100644 index 0000000000000..2ca03e439a0be --- /dev/null +++ b/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-abort.diff @@ -0,0 +1,54 @@ +- // MIR for `multiple_match` before JumpThreading ++ // MIR for `multiple_match` after JumpThreading + + fn multiple_match(_1: u8) -> u8 { + let mut _0: u8; + let mut _2: u8; + let mut _3: u8; + + bb0: { + switchInt(_1) -> [3: bb1, otherwise: bb2]; + } + + bb1: { + _2 = _1; +- switchInt(_2) -> [3: bb3, otherwise: bb4]; ++ goto -> bb3; + } + + bb2: { + _3 = _1; +- switchInt(_3) -> [3: bb5, otherwise: bb6]; ++ goto -> bb6; + } + + bb3: { + _0 = const 5_u8; + return; + } + + bb4: { + _0 = const 7_u8; + return; + } + + bb5: { + _0 = const 9_u8; + return; + } + + bb6: { + switchInt(_3) -> [1: bb7, otherwise: bb8]; + } + + bb7: { + _0 = const 9_u8; + return; + } + + bb8: { + _0 = const 11_u8; + return; + } + } + diff --git a/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-unwind.diff new file mode 100644 index 0000000000000..2ca03e439a0be --- /dev/null +++ b/tests/mir-opt/jump_threading.multiple_match.JumpThreading.panic-unwind.diff @@ -0,0 +1,54 @@ +- // MIR for `multiple_match` before JumpThreading ++ // MIR for `multiple_match` after JumpThreading + + fn multiple_match(_1: u8) -> u8 { + let mut _0: u8; + let mut _2: u8; + let mut _3: u8; + + bb0: { + switchInt(_1) -> [3: bb1, otherwise: bb2]; + } + + bb1: { + _2 = _1; +- switchInt(_2) -> [3: bb3, otherwise: bb4]; ++ goto -> bb3; + } + + bb2: { + _3 = _1; +- switchInt(_3) -> [3: bb5, otherwise: bb6]; ++ goto -> bb6; + } + + bb3: { + _0 = const 5_u8; + return; + } + + bb4: { + _0 = const 7_u8; + return; + } + + bb5: { + _0 = const 9_u8; + return; + } + + bb6: { + switchInt(_3) -> [1: bb7, otherwise: bb8]; + } + + bb7: { + _0 = const 9_u8; + return; + } + + bb8: { + _0 = const 11_u8; + return; + } + } + diff --git a/tests/mir-opt/jump_threading.mutable_ref.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.mutable_ref.JumpThreading.panic-abort.diff new file mode 100644 index 0000000000000..80a422636431e --- /dev/null +++ b/tests/mir-opt/jump_threading.mutable_ref.JumpThreading.panic-abort.diff @@ -0,0 +1,56 @@ +- // MIR for `mutable_ref` before JumpThreading ++ // MIR for `mutable_ref` after JumpThreading + + fn mutable_ref() -> bool { + let mut _0: bool; + let mut _1: i32; + let _3: (); + let mut _4: bool; + let mut _5: i32; + scope 1 { + debug x => _1; + let _2: *mut i32; + scope 2 { + debug a => _2; + scope 3 { + } + } + } + + bb0: { + StorageLive(_1); + _1 = const 5_i32; + StorageLive(_2); + _2 = &raw mut _1; + _1 = const 7_i32; + StorageLive(_3); + (*_2) = const 8_i32; + _3 = const (); + StorageDead(_3); + StorageLive(_4); + StorageLive(_5); + _5 = _1; + _4 = Eq(move _5, const 7_i32); + switchInt(move _4) -> [0: bb2, otherwise: bb1]; + } + + bb1: { + StorageDead(_5); + _0 = const true; + goto -> bb3; + } + + bb2: { + StorageDead(_5); + _0 = const false; + goto -> bb3; + } + + bb3: { + StorageDead(_4); + StorageDead(_2); + StorageDead(_1); + return; + } + } + diff --git a/tests/mir-opt/jump_threading.mutable_ref.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.mutable_ref.JumpThreading.panic-unwind.diff new file mode 100644 index 0000000000000..80a422636431e --- /dev/null +++ b/tests/mir-opt/jump_threading.mutable_ref.JumpThreading.panic-unwind.diff @@ -0,0 +1,56 @@ +- // MIR for `mutable_ref` before JumpThreading ++ // MIR for `mutable_ref` after JumpThreading + + fn mutable_ref() -> bool { + let mut _0: bool; + let mut _1: i32; + let _3: (); + let mut _4: bool; + let mut _5: i32; + scope 1 { + debug x => _1; + let _2: *mut i32; + scope 2 { + debug a => _2; + scope 3 { + } + } + } + + bb0: { + StorageLive(_1); + _1 = const 5_i32; + StorageLive(_2); + _2 = &raw mut _1; + _1 = const 7_i32; + StorageLive(_3); + (*_2) = const 8_i32; + _3 = const (); + StorageDead(_3); + StorageLive(_4); + StorageLive(_5); + _5 = _1; + _4 = Eq(move _5, const 7_i32); + switchInt(move _4) -> [0: bb2, otherwise: bb1]; + } + + bb1: { + StorageDead(_5); + _0 = const true; + goto -> bb3; + } + + bb2: { + StorageDead(_5); + _0 = const false; + goto -> bb3; + } + + bb3: { + StorageDead(_4); + StorageDead(_2); + StorageDead(_1); + return; + } + } + diff --git a/tests/mir-opt/jump_threading.mutate_discriminant.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.mutate_discriminant.JumpThreading.panic-abort.diff new file mode 100644 index 0000000000000..8821b47c34576 --- /dev/null +++ b/tests/mir-opt/jump_threading.mutate_discriminant.JumpThreading.panic-abort.diff @@ -0,0 +1,26 @@ +- // MIR for `mutate_discriminant` before JumpThreading ++ // MIR for `mutate_discriminant` after JumpThreading + + fn mutate_discriminant() -> u8 { + let mut _0: u8; + let mut _1: std::option::Option; + let mut _2: isize; + + bb0: { + discriminant(_1) = 1; + (((_1 as variant#1).0: NonZeroUsize).0: usize) = const 0_usize; + _2 = discriminant(_1); + switchInt(_2) -> [0: bb1, otherwise: bb2]; + } + + bb1: { + _0 = const 1_u8; + return; + } + + bb2: { + _0 = const 2_u8; + unreachable; + } + } + diff --git a/tests/mir-opt/jump_threading.mutate_discriminant.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.mutate_discriminant.JumpThreading.panic-unwind.diff new file mode 100644 index 0000000000000..8821b47c34576 --- /dev/null +++ b/tests/mir-opt/jump_threading.mutate_discriminant.JumpThreading.panic-unwind.diff @@ -0,0 +1,26 @@ +- // MIR for `mutate_discriminant` before JumpThreading ++ // MIR for `mutate_discriminant` after JumpThreading + + fn mutate_discriminant() -> u8 { + let mut _0: u8; + let mut _1: std::option::Option; + let mut _2: isize; + + bb0: { + discriminant(_1) = 1; + (((_1 as variant#1).0: NonZeroUsize).0: usize) = const 0_usize; + _2 = discriminant(_1); + switchInt(_2) -> [0: bb1, otherwise: bb2]; + } + + bb1: { + _0 = const 1_u8; + return; + } + + bb2: { + _0 = const 2_u8; + unreachable; + } + } + diff --git a/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff new file mode 100644 index 0000000000000..20f6861e8db34 --- /dev/null +++ b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff @@ -0,0 +1,53 @@ +- // MIR for `renumbered_bb` before JumpThreading ++ // MIR for `renumbered_bb` after JumpThreading + + fn renumbered_bb(_1: bool) -> u8 { + let mut _0: u8; + let mut _2: bool; + let mut _3: bool; + + bb0: { + _3 = const false; + switchInt(_1) -> [1: bb1, otherwise: bb2]; + } + + bb1: { + _2 = const false; +- goto -> bb3; ++ goto -> bb8; + } + + bb2: { + _2 = _1; + _3 = _1; + goto -> bb3; + } + + bb3: { + switchInt(_2) -> [0: bb4, otherwise: bb5]; + } + + bb4: { + switchInt(_3) -> [0: bb6, otherwise: bb7]; + } + + bb5: { + _0 = const 7_u8; + return; + } + + bb6: { + _0 = const 9_u8; + return; + } + + bb7: { + _0 = const 11_u8; + return; ++ } ++ ++ bb8: { ++ goto -> bb4; + } + } + diff --git a/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff new file mode 100644 index 0000000000000..20f6861e8db34 --- /dev/null +++ b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff @@ -0,0 +1,53 @@ +- // MIR for `renumbered_bb` before JumpThreading ++ // MIR for `renumbered_bb` after JumpThreading + + fn renumbered_bb(_1: bool) -> u8 { + let mut _0: u8; + let mut _2: bool; + let mut _3: bool; + + bb0: { + _3 = const false; + switchInt(_1) -> [1: bb1, otherwise: bb2]; + } + + bb1: { + _2 = const false; +- goto -> bb3; ++ goto -> bb8; + } + + bb2: { + _2 = _1; + _3 = _1; + goto -> bb3; + } + + bb3: { + switchInt(_2) -> [0: bb4, otherwise: bb5]; + } + + bb4: { + switchInt(_3) -> [0: bb6, otherwise: bb7]; + } + + bb5: { + _0 = const 7_u8; + return; + } + + bb6: { + _0 = const 9_u8; + return; + } + + bb7: { + _0 = const 11_u8; + return; ++ } ++ ++ bb8: { ++ goto -> bb4; + } + } + diff --git a/tests/mir-opt/jump_threading.rs b/tests/mir-opt/jump_threading.rs new file mode 100644 index 0000000000000..00dd98d825fdf --- /dev/null +++ b/tests/mir-opt/jump_threading.rs @@ -0,0 +1,291 @@ +// unit-test: JumpThreading +// compile-flags: -Zmir-enable-passes=+Inline +// EMIT_MIR_FOR_EACH_PANIC_STRATEGY + +#![feature(control_flow_enum)] +#![feature(try_trait_v2)] +#![feature(custom_mir, core_intrinsics, rustc_attrs)] + +use std::intrinsics::mir::*; +use std::ops::ControlFlow; + +fn too_complex(x: Result) -> Option { + match { + match x { + Ok(v) => ControlFlow::Continue(v), + Err(r) => ControlFlow::Break(r), + } + } { + ControlFlow::Continue(v) => Some(v), + ControlFlow::Break(r) => None, + } +} + +fn identity(x: Result) -> Result { + Ok(x?) +} + +enum DFA { + A, + B, + C, + D, +} + +fn dfa() { + let mut state = DFA::A; + loop { + match state { + DFA::A => state = DFA::B, + DFA::B => state = DFA::C, + DFA::C => state = DFA::D, + DFA::D => return, + } + } +} + +#[repr(u8)] +enum CustomDiscr { + A = 35, + B = 73, + C = 99, +} + +fn custom_discr(x: bool) -> u8 { + match if x { CustomDiscr::A } else { CustomDiscr::B } { + CustomDiscr::A => 5, + _ => 13, + } +} + +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +fn multiple_match(x: u8) -> u8 { + mir!( + { + match x { 3 => bb1, _ => bb2 } + } + bb1 = { + // We know `x == 3`, so we can take `bb3`. + let y = x; + match y { 3 => bb3, _ => bb4 } + } + bb2 = { + // We know `x != 3`, so we can take `bb6`. + let z = x; + match z { 3 => bb5, _ => bb6 } + } + bb3 = { + RET = 5; + Return() + } + bb4 = { + RET = 7; + Return() + } + bb5 = { + RET = 9; + Return() + } + bb6 = { + // We know `z != 3`, so we CANNOT take `bb7`. + match z { 1 => bb7, _ => bb8 } + } + bb7 = { + RET = 9; + Return() + } + bb8 = { + RET = 11; + Return() + } + ) +} + +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +fn duplicate_chain(x: bool) -> u8 { + mir!( + let a: u8; + { + match x { true => bb1, _ => bb2 } + } + bb1 = { + a = 5; + Goto(bb3) + } + bb2 = { + a = 5; + Goto(bb3) + } + // Verify that we do not create multiple copied of `bb3`. + bb3 = { + let b = 13; + Goto(bb4) + } + bb4 = { + let c = 15; + match a { 5 => bb5, _ => bb6 } + } + bb5 = { + RET = 7; + Return() + } + bb6 = { + RET = 9; + Return() + } + ) +} + +#[rustc_layout_scalar_valid_range_start(1)] +#[rustc_nonnull_optimization_guaranteed] +struct NonZeroUsize(usize); + +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +fn mutate_discriminant() -> u8 { + mir!( + let x: Option; + { + SetDiscriminant(x, 1); + // This assignment overwrites the niche in which the discriminant is stored. + place!(Field(Field(Variant(x, 1), 0), 0)) = 0_usize; + // So we cannot know the value of this discriminant. + let a = Discriminant(x); + match a { + 0 => bb1, + _ => bad, + } + } + bb1 = { + RET = 1; + Return() + } + bad = { + RET = 2; + Unreachable() + } + ) +} + +// Verify that we do not try to reason when there are mutable pointers involved. +fn mutable_ref() -> bool { + let mut x = 5; + let a = std::ptr::addr_of_mut!(x); + x = 7; + unsafe { *a = 8 }; + if x == 7 { + true + } else { + false + } +} + +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +fn renumbered_bb(x: bool) -> u8 { + // This function has 2 TOs: 1-3-4 and 0-1-3-4-6. + // We verify that the second TO does not modify 3 once the first has been applied. + mir!( + let a: bool; + let b: bool; + { + b = false; + match x { true => bb1, _ => bb2 } + } + bb1 = { + a = false; + Goto(bb3) + } + bb2 = { + a = x; + b = x; + Goto(bb3) + } + bb3 = { + match a { false => bb4, _ => bb5 } + } + bb4 = { + match b { false => bb6, _ => bb7 } + } + bb5 = { + RET = 7; + Return() + } + bb6 = { + RET = 9; + Return() + } + bb7 = { + RET = 11; + Return() + } + ) +} + +#[custom_mir(dialect = "runtime", phase = "post-cleanup")] +fn disappearing_bb(x: u8) -> u8 { + // This function has 3 TOs: 1-4-5, 0-1-4-7-5-8 and 3-4-7-5-6 + // After applying the first TO, we create bb9 to replace 4, and rename 1-4 edge by 1-9. The + // second TO may try to thread non-existing edge 9-4. + // This test verifies that we preserve semantics by bailing out of this second TO. + mir!( + let _11: i8; + let _12: bool; + let _13: bool; + { + _13 = false; + _12 = false; + _13 = true; + _12 = true; + match x { 0 => bb3, 1 => bb3, 2 => bb1, _ => bb2 } + } + bb1 = { + _12 = false; + Goto(bb4) + } + bb2 = { + Unreachable() + } + bb3 = { + _13 = false; + Goto(bb4) + } + bb4 = { + match _12 { false => bb5, _ => bb7 } + } + bb5 = { + match _13 { false => bb6, _ => bb8 } + } + bb6 = { + Return() + } + bb7 = { + Goto(bb5) + } + bb8 = { + Goto(bb6) + } + ) +} + +fn main() { + too_complex(Ok(0)); + identity(Ok(0)); + custom_discr(false); + dfa(); + multiple_match(5); + duplicate_chain(false); + mutate_discriminant(); + mutable_ref(); + renumbered_bb(true); + disappearing_bb(7); +} + +// EMIT_MIR jump_threading.too_complex.JumpThreading.diff +// EMIT_MIR jump_threading.identity.JumpThreading.diff +// EMIT_MIR jump_threading.custom_discr.JumpThreading.diff +// EMIT_MIR jump_threading.dfa.JumpThreading.diff +// EMIT_MIR jump_threading.multiple_match.JumpThreading.diff +// EMIT_MIR jump_threading.duplicate_chain.JumpThreading.diff +// EMIT_MIR jump_threading.mutate_discriminant.JumpThreading.diff +// EMIT_MIR jump_threading.mutable_ref.JumpThreading.diff +// EMIT_MIR jump_threading.renumbered_bb.JumpThreading.diff +// EMIT_MIR jump_threading.disappearing_bb.JumpThreading.diff diff --git a/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-abort.diff new file mode 100644 index 0000000000000..f5eade4a9149e --- /dev/null +++ b/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-abort.diff @@ -0,0 +1,98 @@ +- // MIR for `too_complex` before JumpThreading ++ // MIR for `too_complex` after JumpThreading + + fn too_complex(_1: Result) -> Option { + debug x => _1; + let mut _0: std::option::Option; + let mut _2: std::ops::ControlFlow; + let mut _3: isize; + let _4: i32; + let mut _5: i32; + let _6: usize; + let mut _7: usize; + let mut _8: isize; + let _9: i32; + let mut _10: i32; + let _11: usize; + scope 1 { + debug v => _4; + } + scope 2 { + debug r => _6; + } + scope 3 { + debug v => _9; + } + scope 4 { + debug r => _11; + } + + bb0: { + StorageLive(_2); + _3 = discriminant(_1); + switchInt(move _3) -> [0: bb3, 1: bb1, otherwise: bb2]; + } + + bb1: { + StorageLive(_6); + _6 = ((_1 as Err).0: usize); + StorageLive(_7); + _7 = _6; + _2 = ControlFlow::::Break(move _7); + StorageDead(_7); + StorageDead(_6); +- goto -> bb4; ++ goto -> bb8; + } + + bb2: { + unreachable; + } + + bb3: { + StorageLive(_4); + _4 = ((_1 as Ok).0: i32); + StorageLive(_5); + _5 = _4; + _2 = ControlFlow::::Continue(move _5); + StorageDead(_5); + StorageDead(_4); + goto -> bb4; + } + + bb4: { + _8 = discriminant(_2); +- switchInt(move _8) -> [0: bb6, 1: bb5, otherwise: bb2]; ++ goto -> bb6; + } + + bb5: { + StorageLive(_11); + _11 = ((_2 as Break).0: usize); + _0 = Option::::None; + StorageDead(_11); + goto -> bb7; + } + + bb6: { + StorageLive(_9); + _9 = ((_2 as Continue).0: i32); + StorageLive(_10); + _10 = _9; + _0 = Option::::Some(move _10); + StorageDead(_10); + StorageDead(_9); + goto -> bb7; + } + + bb7: { + StorageDead(_2); + return; ++ } ++ ++ bb8: { ++ _8 = discriminant(_2); ++ goto -> bb5; + } + } + diff --git a/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-unwind.diff new file mode 100644 index 0000000000000..f5eade4a9149e --- /dev/null +++ b/tests/mir-opt/jump_threading.too_complex.JumpThreading.panic-unwind.diff @@ -0,0 +1,98 @@ +- // MIR for `too_complex` before JumpThreading ++ // MIR for `too_complex` after JumpThreading + + fn too_complex(_1: Result) -> Option { + debug x => _1; + let mut _0: std::option::Option; + let mut _2: std::ops::ControlFlow; + let mut _3: isize; + let _4: i32; + let mut _5: i32; + let _6: usize; + let mut _7: usize; + let mut _8: isize; + let _9: i32; + let mut _10: i32; + let _11: usize; + scope 1 { + debug v => _4; + } + scope 2 { + debug r => _6; + } + scope 3 { + debug v => _9; + } + scope 4 { + debug r => _11; + } + + bb0: { + StorageLive(_2); + _3 = discriminant(_1); + switchInt(move _3) -> [0: bb3, 1: bb1, otherwise: bb2]; + } + + bb1: { + StorageLive(_6); + _6 = ((_1 as Err).0: usize); + StorageLive(_7); + _7 = _6; + _2 = ControlFlow::::Break(move _7); + StorageDead(_7); + StorageDead(_6); +- goto -> bb4; ++ goto -> bb8; + } + + bb2: { + unreachable; + } + + bb3: { + StorageLive(_4); + _4 = ((_1 as Ok).0: i32); + StorageLive(_5); + _5 = _4; + _2 = ControlFlow::::Continue(move _5); + StorageDead(_5); + StorageDead(_4); + goto -> bb4; + } + + bb4: { + _8 = discriminant(_2); +- switchInt(move _8) -> [0: bb6, 1: bb5, otherwise: bb2]; ++ goto -> bb6; + } + + bb5: { + StorageLive(_11); + _11 = ((_2 as Break).0: usize); + _0 = Option::::None; + StorageDead(_11); + goto -> bb7; + } + + bb6: { + StorageLive(_9); + _9 = ((_2 as Continue).0: i32); + StorageLive(_10); + _10 = _9; + _0 = Option::::Some(move _10); + StorageDead(_10); + StorageDead(_9); + goto -> bb7; + } + + bb7: { + StorageDead(_2); + return; ++ } ++ ++ bb8: { ++ _8 = discriminant(_2); ++ goto -> bb5; + } + } + From 0d0a5367772aacd4c0baf954f601952a3a2017aa Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Thu, 20 Apr 2023 17:07:16 +0000 Subject: [PATCH 04/16] Do not thread through loop headers. --- .../rustc_mir_transform/src/jump_threading.rs | 45 +++++++++++++++++-- ...reading.dfa.JumpThreading.panic-abort.diff | 19 ++------ ...eading.dfa.JumpThreading.panic-unwind.diff | 19 ++------ 3 files changed, 48 insertions(+), 35 deletions(-) diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 1831184aedae5..0046d6d694140 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -26,11 +26,14 @@ //! - bound the maximum depth by a constant `MAX_BACKTRACK`; //! - we only traverse `Goto` terminators. //! +//! We try to avoid creating irreducible control-flow by not threading through a loop header. +//! //! Likewise, applying the optimisation can create a lot of new MIR, so we bound the instruction //! cost by `MAX_COST`. use rustc_arena::DroplessArena; use rustc_data_structures::fx::FxHashSet; +use rustc_index::bit_set::BitSet; use rustc_index::IndexVec; use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::*; @@ -58,14 +61,22 @@ impl<'tcx> MirPass<'tcx> for JumpThreading { let param_env = tcx.param_env_reveal_all_normalized(def_id); let map = Map::new(tcx, body, Some(MAX_PLACES)); + let loop_headers = loop_headers(body); let arena = DroplessArena::default(); - let mut finder = - TOFinder { tcx, param_env, body, arena: &arena, map: &map, opportunities: Vec::new() }; + let mut finder = TOFinder { + tcx, + param_env, + body, + arena: &arena, + map: &map, + loop_headers: &loop_headers, + opportunities: Vec::new(), + }; for (bb, bbdata) in body.basic_blocks.iter_enumerated() { debug!(?bb, term = ?bbdata.terminator()); - if bbdata.is_cleanup { + if bbdata.is_cleanup || loop_headers.contains(bb) { continue; } let Some((discr, targets)) = bbdata.terminator().kind.as_switch() else { continue }; @@ -108,6 +119,10 @@ impl<'tcx> MirPass<'tcx> for JumpThreading { return; } + // Verify that we do not thread through a loop header. + for to in opportunities.iter() { + assert!(to.chain.iter().all(|&block| !loop_headers.contains(block))); + } OpportunitySet::new(body, opportunities).apply(body); } } @@ -125,6 +140,7 @@ struct TOFinder<'tcx, 'a> { param_env: ty::ParamEnv<'tcx>, body: &'a Body<'tcx>, map: &'a Map, + loop_headers: &'a BitSet, /// We use an arena to avoid cloning the slices when cloning `state`. arena: &'a DroplessArena, opportunities: Vec, @@ -180,6 +196,11 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { mut cost: CostChecker<'_, 'tcx>, depth: usize, ) { + // Do not thread through loop headers. + if self.loop_headers.contains(bb) { + return; + } + debug!(cost = ?cost.cost()); for (statement_index, stmt) in self.body.basic_blocks[bb].statements.iter().enumerate().rev() @@ -636,3 +657,21 @@ enum Update { Incr, Decr, } + +/// Compute the set of loop headers in the given body. We define a loop header as a block which has +/// at least a predecessor which it dominates. This definition is only correct for reducible CFGs. +/// But if the CFG is already irreducible, there is no point in trying much harder. +/// is already irreducibl +fn loop_headers(body: &Body<'_>) -> BitSet { + let mut loop_headers = BitSet::new_empty(body.basic_blocks.len()); + let dominators = body.basic_blocks.dominators(); + // Only visit reachable blocks. + for (bb, bbdata) in traversal::preorder(body) { + for succ in bbdata.terminator().successors() { + if dominators.dominates(succ, bb) { + loop_headers.insert(bb); + } + } + } + loop_headers +} diff --git a/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-abort.diff index bad733268a30e..01bddc9a173f4 100644 --- a/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-abort.diff @@ -25,8 +25,7 @@ bb1: { _4 = discriminant(_1); -- switchInt(move _4) -> [0: bb4, 1: bb5, 2: bb6, 3: bb2, otherwise: bb3]; -+ goto -> bb2; + switchInt(move _4) -> [0: bb4, 1: bb5, 2: bb6, 3: bb2, otherwise: bb3]; } bb2: { @@ -46,8 +45,7 @@ _1 = move _5; _3 = const (); StorageDead(_5); -- goto -> bb1; -+ goto -> bb8; + goto -> bb1; } bb5: { @@ -56,8 +54,7 @@ _1 = move _6; _3 = const (); StorageDead(_6); -- goto -> bb1; -+ goto -> bb9; + goto -> bb1; } bb6: { @@ -72,16 +69,6 @@ + bb7: { + _4 = discriminant(_1); + goto -> bb4; -+ } -+ -+ bb8: { -+ _4 = discriminant(_1); -+ goto -> bb5; -+ } -+ -+ bb9: { -+ _4 = discriminant(_1); -+ goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-unwind.diff index bad733268a30e..01bddc9a173f4 100644 --- a/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-unwind.diff @@ -25,8 +25,7 @@ bb1: { _4 = discriminant(_1); -- switchInt(move _4) -> [0: bb4, 1: bb5, 2: bb6, 3: bb2, otherwise: bb3]; -+ goto -> bb2; + switchInt(move _4) -> [0: bb4, 1: bb5, 2: bb6, 3: bb2, otherwise: bb3]; } bb2: { @@ -46,8 +45,7 @@ _1 = move _5; _3 = const (); StorageDead(_5); -- goto -> bb1; -+ goto -> bb8; + goto -> bb1; } bb5: { @@ -56,8 +54,7 @@ _1 = move _6; _3 = const (); StorageDead(_6); -- goto -> bb1; -+ goto -> bb9; + goto -> bb1; } bb6: { @@ -72,16 +69,6 @@ + bb7: { + _4 = discriminant(_1); + goto -> bb4; -+ } -+ -+ bb8: { -+ _4 = discriminant(_1); -+ goto -> bb5; -+ } -+ -+ bb9: { -+ _4 = discriminant(_1); -+ goto -> bb6; } } From df9e5ee03896889ed20d9dc8ebecaed1a429a776 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Tue, 7 Feb 2023 22:02:43 +0000 Subject: [PATCH 05/16] Handle more terminators. --- .../rustc_mir_transform/src/jump_threading.rs | 92 ++++++++++++++----- ...ppearing_bb.JumpThreading.panic-abort.diff | 9 +- ...pearing_bb.JumpThreading.panic-unwind.diff | 9 +- ...numbered_bb.JumpThreading.panic-abort.diff | 6 +- ...umbered_bb.JumpThreading.panic-unwind.diff | 6 +- 5 files changed, 95 insertions(+), 27 deletions(-) diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 0046d6d694140..6558b04d4bfd9 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -234,21 +234,17 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { let predecessors = &self.body.basic_blocks.predecessors()[bb]; if let &[pred] = &predecessors[..] && bb != START_BLOCK { - match &self.body.basic_blocks[pred].terminator().kind { - TerminatorKind::Goto { .. } => self.find_opportunity(pred, state, cost, depth), - TerminatorKind::SwitchInt { discr, targets } => { - self.process_switch_int(state, discr, targets, bb); + let term = self.body.basic_blocks[pred].terminator(); + match term.kind { + TerminatorKind::SwitchInt { ref discr, ref targets } => { + self.process_switch_int(discr, targets, bb, &mut state); + self.find_opportunity(pred, state, cost, depth + 1); } - _ => {} + _ => self.recurse_through_terminator(pred, &state, &cost, depth), } } else { for &pred in predecessors { - if matches!( - self.body.basic_blocks[pred].terminator().kind, - TerminatorKind::Goto { .. } - ) { - self.find_opportunity(pred, state.clone(), cost.clone(), depth + 1); - } + self.recurse_through_terminator(pred, &state, &cost, depth); } } @@ -464,35 +460,89 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { None } + #[instrument(level = "trace", skip(self, cost))] + fn recurse_through_terminator( + &mut self, + bb: BasicBlock, + state: &State>, + cost: &CostChecker<'_, 'tcx>, + depth: usize, + ) { + let register_opportunity = |c: Condition| { + debug!(?bb, ?c.target, "register"); + self.opportunities.push(ThreadingOpportunity { chain: vec![bb], target: c.target }) + }; + + let term = self.body.basic_blocks[bb].terminator(); + let place_to_flood = match term.kind { + // We come from a target, so those are not possible. + TerminatorKind::UnwindResume + | TerminatorKind::UnwindTerminate(_) + | TerminatorKind::Return + | TerminatorKind::Unreachable + | TerminatorKind::CoroutineDrop => bug!("{term:?} has no terminators"), + // Disallowed during optimizations. + TerminatorKind::FalseEdge { .. } + | TerminatorKind::FalseUnwind { .. } + | TerminatorKind::Yield { .. } => bug!("{term:?} invalid"), + // Cannot reason about inline asm. + TerminatorKind::InlineAsm { .. } => return, + // `SwitchInt` is handled specially. + TerminatorKind::SwitchInt { .. } => return, + // We can recurse, no thing particular to do. + TerminatorKind::Goto { .. } => None, + // Flood the overwritten place, and progress through. + TerminatorKind::Drop { place: destination, .. } + | TerminatorKind::Call { destination, .. } => Some(destination), + // Treat as an `assume(cond == expected)`. + TerminatorKind::Assert { ref cond, expected, .. } => { + if let Some(place) = cond.place() + && let Some(conditions) = state.try_get(place.as_ref(), self.map) + { + let expected = if expected { ScalarInt::TRUE } else { ScalarInt::FALSE }; + conditions.iter_matches(expected).for_each(register_opportunity); + } + None + } + }; + + // We can recurse through this terminator. + let mut state = state.clone(); + if let Some(place_to_flood) = place_to_flood { + state.flood_with(place_to_flood.as_ref(), self.map, ConditionSet::default()); + } + self.find_opportunity(bb, state, cost.clone(), depth + 1); + } + #[instrument(level = "trace", skip(self))] fn process_switch_int( &mut self, - state: State>, discr: &Operand<'tcx>, targets: &SwitchTargets, - bb: BasicBlock, + target_bb: BasicBlock, + state: &mut State>, ) -> Option { - debug_assert_ne!(bb, START_BLOCK); - debug_assert_eq!(self.body.basic_blocks.predecessors()[bb].len(), 1); + debug_assert_ne!(target_bb, START_BLOCK); + debug_assert_eq!(self.body.basic_blocks.predecessors()[target_bb].len(), 1); let discr = discr.place()?; let discr_ty = discr.ty(self.body, self.tcx).ty; let discr_layout = self.tcx.layout_of(self.param_env.and(discr_ty)).ok()?; let conditions = state.try_get(discr.as_ref(), self.map)?; - if let Some((value, _)) = targets.iter().find(|&(_, target)| target == bb) { + if let Some((value, _)) = targets.iter().find(|&(_, target)| target == target_bb) { let value = ScalarInt::try_from_uint(value, discr_layout.size)?; - debug_assert_eq!(targets.iter().filter(|&(_, target)| target == bb).count(), 1); + debug_assert_eq!(targets.iter().filter(|&(_, target)| target == target_bb).count(), 1); - // We are inside `bb`. Since we have a single predecessor, we know we passed + // We are inside `target_bb`. Since we have a single predecessor, we know we passed // through the `SwitchInt` before arriving here. Therefore, we know that // `discr == value`. If one condition can be fulfilled by `discr == value`, // that's an opportunity. for c in conditions.iter_matches(value) { - debug!(?bb, ?c.target, "register"); + debug!(?target_bb, ?c.target, "register"); self.opportunities.push(ThreadingOpportunity { chain: vec![], target: c.target }); } - } else if bb == targets.otherwise() { + } else if target_bb == targets.otherwise() { let (value, _, _) = targets.as_static_if()?; let value = ScalarInt::try_from_uint(value, discr_layout.size)?; @@ -500,7 +550,7 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { // so we can only match the exact same condition. for c in conditions.iter() { if c.value == value && c.polarity == false { - debug!(?bb, ?c.target, "register"); + debug!(?target_bb, ?c.target, "register"); self.opportunities .push(ThreadingOpportunity { chain: vec![], target: c.target }); } diff --git a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff index d3f737b324c93..9c84aff536107 100644 --- a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff @@ -43,15 +43,20 @@ } bb7: { - goto -> bb5; +- goto -> bb5; ++ goto -> bb10; } bb8: { - goto -> bb6; ++ goto -> bb6; + } + + bb9: { + goto -> bb5; ++ } ++ ++ bb10: { + goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff index d3f737b324c93..9c84aff536107 100644 --- a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff @@ -43,15 +43,20 @@ } bb7: { - goto -> bb5; +- goto -> bb5; ++ goto -> bb10; } bb8: { - goto -> bb6; ++ goto -> bb6; + } + + bb9: { + goto -> bb5; ++ } ++ ++ bb10: { + goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff index 20f6861e8db34..2d943a4bee219 100644 --- a/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-abort.diff @@ -47,7 +47,11 @@ + } + + bb8: { -+ goto -> bb4; ++ goto -> bb9; ++ } ++ ++ bb9: { ++ goto -> bb6; } } diff --git a/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff index 20f6861e8db34..2d943a4bee219 100644 --- a/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.renumbered_bb.JumpThreading.panic-unwind.diff @@ -47,7 +47,11 @@ + } + + bb8: { -+ goto -> bb4; ++ goto -> bb9; ++ } ++ ++ bb9: { ++ goto -> bb6; } } From 4abea8366339b66932bf80203dd0884366fdb872 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Sat, 14 Oct 2023 13:26:14 +0000 Subject: [PATCH 06/16] Improve naming and comments. --- .../rustc_mir_dataflow/src/value_analysis.rs | 62 ++++++++++++++----- .../rustc_mir_transform/src/jump_threading.rs | 2 +- 2 files changed, 46 insertions(+), 18 deletions(-) diff --git a/compiler/rustc_mir_dataflow/src/value_analysis.rs b/compiler/rustc_mir_dataflow/src/value_analysis.rs index 77c9a22df79ce..025d2ddfd4ff7 100644 --- a/compiler/rustc_mir_dataflow/src/value_analysis.rs +++ b/compiler/rustc_mir_dataflow/src/value_analysis.rs @@ -496,10 +496,12 @@ impl State { values.raw.fill(value); } + /// Assign `value` to all places that are contained in `place` or may alias one. pub fn flood_with(&mut self, place: PlaceRef<'_>, map: &Map, value: V) { - self.flood_with_extra(place, None, map, value) + self.flood_with_tail_elem(place, None, map, value) } + /// Assign `TOP` to all places that are contained in `place` or may alias one. pub fn flood(&mut self, place: PlaceRef<'_>, map: &Map) where V: HasTop, @@ -507,10 +509,12 @@ impl State { self.flood_with(place, map, V::TOP) } + /// Assign `value` to the discriminant of `place` and all places that may alias it. pub fn flood_discr_with(&mut self, place: PlaceRef<'_>, map: &Map, value: V) { - self.flood_with_extra(place, Some(TrackElem::Discriminant), map, value) + self.flood_with_tail_elem(place, Some(TrackElem::Discriminant), map, value) } + /// Assign `TOP` to the discriminant of `place` and all places that may alias it. pub fn flood_discr(&mut self, place: PlaceRef<'_>, map: &Map) where V: HasTop, @@ -518,7 +522,14 @@ impl State { self.flood_discr_with(place, map, V::TOP) } - pub fn flood_with_extra( + /// This method is the most general version of the `flood_*` method. + /// + /// Assign `value` on the given place and all places that may alias it. In particular, when + /// the given place has a variant downcast, we invoke the function on all the other variants. + /// + /// `tail_elem` allows to support discriminants that are not a place in MIR, but that we track + /// as such. + pub fn flood_with_tail_elem( &mut self, place: PlaceRef<'_>, tail_elem: Option, @@ -602,62 +613,79 @@ impl State { } } - /// Retrieve the value stored for a place, or ⊤ if it is not tracked. + /// Retrieve the value stored for a place, or `None` if it is not tracked. pub fn try_get(&self, place: PlaceRef<'_>, map: &Map) -> Option { let place = map.find(place)?; self.try_get_idx(place, map) } - /// Retrieve the value stored for a place, or ⊤ if it is not tracked. + /// Retrieve the discriminant stored for a place, or `None` if it is not tracked. pub fn try_get_discr(&self, place: PlaceRef<'_>, map: &Map) -> Option { let place = map.find_discr(place)?; self.try_get_idx(place, map) } - /// Retrieve the value stored for a place index, or ⊤ if it is not tracked. + /// Retrieve the slice length stored for a place, or `None` if it is not tracked. + pub fn try_get_len(&self, place: PlaceRef<'_>, map: &Map) -> Option { + let place = map.find_len(place)?; + self.try_get_idx(place, map) + } + + /// Retrieve the value stored for a place index, or `None` if it is not tracked. pub fn try_get_idx(&self, place: PlaceIndex, map: &Map) -> Option { match &self.0 { StateData::Reachable(values) => { map.places[place].value_index.map(|v| values[v].clone()) } - StateData::Unreachable => { - // Because this is unreachable, we can return any value we want. - None - } + StateData::Unreachable => None, } } /// Retrieve the value stored for a place, or ⊤ if it is not tracked. + /// + /// This method returns ⊥ if the place is tracked and the state is unreachable. pub fn get(&self, place: PlaceRef<'_>, map: &Map) -> V where V: HasBottom + HasTop, { - map.find(place).map(|place| self.get_idx(place, map)).unwrap_or(V::TOP) + match &self.0 { + StateData::Reachable(_) => self.try_get(place, map).unwrap_or(V::TOP), + // Because this is unreachable, we can return any value we want. + StateData::Unreachable => V::BOTTOM, + } } /// Retrieve the value stored for a place, or ⊤ if it is not tracked. + /// + /// This method returns ⊥ the current state is unreachable. pub fn get_discr(&self, place: PlaceRef<'_>, map: &Map) -> V where V: HasBottom + HasTop, { - match map.find_discr(place) { - Some(place) => self.get_idx(place, map), - None => V::TOP, + match &self.0 { + StateData::Reachable(_) => self.try_get_discr(place, map).unwrap_or(V::TOP), + // Because this is unreachable, we can return any value we want. + StateData::Unreachable => V::BOTTOM, } } /// Retrieve the value stored for a place, or ⊤ if it is not tracked. + /// + /// This method returns ⊥ the current state is unreachable. pub fn get_len(&self, place: PlaceRef<'_>, map: &Map) -> V where V: HasBottom + HasTop, { - match map.find_len(place) { - Some(place) => self.get_idx(place, map), - None => V::TOP, + match &self.0 { + StateData::Reachable(_) => self.try_get_len(place, map).unwrap_or(V::TOP), + // Because this is unreachable, we can return any value we want. + StateData::Unreachable => V::BOTTOM, } } /// Retrieve the value stored for a place index, or ⊤ if it is not tracked. + /// + /// This method returns ⊥ the current state is unreachable. pub fn get_idx(&self, place: PlaceIndex, map: &Map) -> V where V: HasBottom + HasTop, diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 6558b04d4bfd9..a5ee843e23ae4 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -222,7 +222,7 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { // _1 = 5 // Whatever happens here, it won't change the result of a `SwitchInt`. // _1 = 6 if let Some((lhs, tail)) = self.mutated_statement(stmt) { - state.flood_with_extra(lhs.as_ref(), tail, self.map, ConditionSet::default()); + state.flood_with_tail_elem(lhs.as_ref(), tail, self.map, ConditionSet::default()); } } From b5aa1ef9b42c744db10e20fa946c6a4bb66b2cbb Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Tue, 17 Oct 2023 18:35:13 +0000 Subject: [PATCH 07/16] Document mutated_statement. --- .../rustc_mir_transform/src/jump_threading.rs | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index a5ee843e23ae4..16da3c934e786 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -273,6 +273,18 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { } /// Extract the mutated place from a statement. + /// + /// This method returns the `Place` so we can flood the state in case of a partial assignment. + /// (_1 as Ok).0 = _5; + /// (_1 as Err).0 = _6; + /// We want to ensure that a `SwitchInt((_1 as Ok).0)` does not see the first assignment, as + /// the value may have been mangled by the second assignment. + /// + /// In case we assign to a discriminant, we return `Some(TrackElem::Discriminant)`, so we can + /// stop at flooding the discriminant, and preserve the variant fields. + /// (_1 as Some).0 = _6; + /// SetDiscriminant(_1, 1); + /// switchInt((_1 as Some).0) #[instrument(level = "trace", skip(self), ret)] fn mutated_statement( &self, @@ -280,9 +292,6 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { ) -> Option<(Place<'tcx>, Option)> { match stmt.kind { StatementKind::Assign(box (place, _)) - | StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume( - Operand::Copy(place) | Operand::Move(place), - )) | StatementKind::Deinit(box place) => Some((place, None)), StatementKind::SetDiscriminant { box place, variant_index: _ } => { Some((place, Some(TrackElem::Discriminant))) @@ -291,7 +300,9 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { Some((Place::from(local), None)) } StatementKind::Retag(..) - | StatementKind::Intrinsic(..) + | StatementKind::Intrinsic(box NonDivergingIntrinsic::Assume(..)) + // copy_nonoverlapping takes pointers and mutated the pointed-to value. + | StatementKind::Intrinsic(box NonDivergingIntrinsic::CopyNonOverlapping(..)) | StatementKind::AscribeUserType(..) | StatementKind::Coverage(..) | StatementKind::FakeRead(..) From 47056248e5289e74252cf501a0e32f336f5a9a04 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Tue, 17 Oct 2023 18:40:06 +0000 Subject: [PATCH 08/16] Make polarity an enum. --- .../rustc_mir_transform/src/jump_threading.rs | 34 +++++++++++++------ 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 16da3c934e786..bb8259d07094c 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -11,9 +11,9 @@ //! looking for assignments that will turn the `SwitchInt` into a simple `Goto`. //! //! The algorithm maintains a set of replacement conditions: -//! - `conditions[place]` contains `Condition { value, polarity: true, target }` +//! - `conditions[place]` contains `Condition { value, polarity: Eq, target }` //! if assigning `value` to `place` turns the `SwitchInt` into `Goto { target }`. -//! - `conditions[place]` contains `Condition { value, polarity: false, target }` +//! - `conditions[place]` contains `Condition { value, polarity: Ne, target }` //! if assigning anything different from `value` to `place` turns the `SwitchInt` //! into `Goto { target }`. //! @@ -98,13 +98,13 @@ impl<'tcx> MirPass<'tcx> for JumpThreading { continue; }; arena.alloc_from_iter([ - Condition { value, polarity: true, target: then }, - Condition { value, polarity: false, target: else_ }, + Condition { value, polarity: Polarity::Eq, target: then }, + Condition { value, polarity: Polarity::Ne, target: else_ }, ]) } else { arena.alloc_from_iter(targets.iter().filter_map(|(value, target)| { let value = ScalarInt::try_from_uint(value, discr_layout.size)?; - Some(Condition { value, polarity: true, target }) + Some(Condition { value, polarity: Polarity::Eq, target }) })) }; let conds = ConditionSet(conds); @@ -149,18 +149,26 @@ struct TOFinder<'tcx, 'a> { #[derive(Copy, Clone, Debug)] struct Condition { value: ScalarInt, - /// `true` means `==`, `false` means `!=` - polarity: bool, + polarity: Polarity, target: BasicBlock, } +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +enum Polarity { + Ne, + Eq, +} + impl Condition { fn matches(&self, value: ScalarInt) -> bool { - (self.value == value) == self.polarity + (self.value == value) == (self.polarity == Polarity::Eq) } fn inv(mut self) -> Self { - self.polarity = !self.polarity; + self.polarity = match self.polarity { + Polarity::Eq => Polarity::Ne, + Polarity::Ne => Polarity::Eq, + }; self } } @@ -455,7 +463,11 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { .try_to_scalar_int()?; let conds = conditions.map(self.arena, |c| Condition { value, - polarity: c.matches(equals), + polarity: if c.matches(equals) { + Polarity::Eq + } else { + Polarity::Ne + }, ..c }); state.insert_value_idx(place, conds, self.map); @@ -560,7 +572,7 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { // Likewise, we know that `discr != value`. That's a must weaker information, // so we can only match the exact same condition. for c in conditions.iter() { - if c.value == value && c.polarity == false { + if c.value == value && c.polarity == Polarity::Ne { debug!(?target_bb, ?c.target, "register"); self.opportunities .push(ThreadingOpportunity { chain: vec![], target: c.target }); From 98b86fa9e5f6edfc696ce80e2b8ca33b73249031 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Tue, 17 Oct 2023 18:45:48 +0000 Subject: [PATCH 09/16] Document Condition. --- compiler/rustc_mir_transform/src/jump_threading.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index bb8259d07094c..701d03de1a20f 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -146,6 +146,8 @@ struct TOFinder<'tcx, 'a> { opportunities: Vec, } +/// Represent the following statement. If we can prove that the current local is equal/not-equal +/// to `value`, jump to `target`. #[derive(Copy, Clone, Debug)] struct Condition { value: ScalarInt, From 6abd8f119e48c707488a545cf2fc0614d7dc433c Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Wed, 18 Oct 2023 16:28:25 +0000 Subject: [PATCH 10/16] Rebase fallout. --- tests/mir-opt/jump_threading.rs | 2 +- ...orage.while_loop.PreCodegen.after.panic-abort.mir | 12 ------------ ...rage.while_loop.PreCodegen.after.panic-unwind.mir | 12 ------------ 3 files changed, 1 insertion(+), 25 deletions(-) diff --git a/tests/mir-opt/jump_threading.rs b/tests/mir-opt/jump_threading.rs index 00dd98d825fdf..dcf6177356544 100644 --- a/tests/mir-opt/jump_threading.rs +++ b/tests/mir-opt/jump_threading.rs @@ -1,5 +1,5 @@ // unit-test: JumpThreading -// compile-flags: -Zmir-enable-passes=+Inline +// compile-flags: -Zmir-enable-passes=+Inline -Zinline-mir // EMIT_MIR_FOR_EACH_PANIC_STRATEGY #![feature(control_flow_enum)] diff --git a/tests/mir-opt/while_storage.while_loop.PreCodegen.after.panic-abort.mir b/tests/mir-opt/while_storage.while_loop.PreCodegen.after.panic-abort.mir index e6ce33ed682f6..21c4b92cf0458 100644 --- a/tests/mir-opt/while_storage.while_loop.PreCodegen.after.panic-abort.mir +++ b/tests/mir-opt/while_storage.while_loop.PreCodegen.after.panic-abort.mir @@ -11,18 +11,6 @@ fn while_loop(_1: bool) -> () { } bb0: { - goto -> bb1; - } - - bb1: { - switchInt(_1) -> [0: bb3, otherwise: bb2]; - } - - bb2: { - switchInt(_1) -> [0: bb1, otherwise: bb3]; - } - - bb3: { return; } } diff --git a/tests/mir-opt/while_storage.while_loop.PreCodegen.after.panic-unwind.mir b/tests/mir-opt/while_storage.while_loop.PreCodegen.after.panic-unwind.mir index e6ce33ed682f6..21c4b92cf0458 100644 --- a/tests/mir-opt/while_storage.while_loop.PreCodegen.after.panic-unwind.mir +++ b/tests/mir-opt/while_storage.while_loop.PreCodegen.after.panic-unwind.mir @@ -11,18 +11,6 @@ fn while_loop(_1: bool) -> () { } bb0: { - goto -> bb1; - } - - bb1: { - switchInt(_1) -> [0: bb3, otherwise: bb2]; - } - - bb2: { - switchInt(_1) -> [0: bb1, otherwise: bb3]; - } - - bb3: { return; } } From d5b21ef96bb9e4824b4651bfcf25a42465767a78 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Wed, 18 Oct 2023 16:41:10 +0000 Subject: [PATCH 11/16] Explicit notation. --- compiler/rustc_mir_transform/src/jump_threading.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 701d03de1a20f..5f0e7fcf2d28b 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -17,6 +17,10 @@ //! if assigning anything different from `value` to `place` turns the `SwitchInt` //! into `Goto { target }`. //! +//! In this file, we denote as `place ?= value` the existence of a replacement condition +//! on `place` with given `value`, irrespective of the polarity and target of that +//! replacement condition. +//! //! We then walk the CFG backwards transforming the set of conditions. //! When we find a fulfilling assignment, we record a `ThreadingOpportunity`. //! All `ThreadingOpportunity`s are applied to the body, by duplicating blocks if required. From 66ec098ea7804ce8d02275cd754ccdcd7813bfdf Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Thu, 19 Oct 2023 16:21:27 +0000 Subject: [PATCH 12/16] Simplify static if handling. --- compiler/rustc_mir_transform/src/jump_threading.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 5f0e7fcf2d28b..19ba403ec6cbf 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -571,12 +571,14 @@ impl<'tcx, 'a> TOFinder<'tcx, 'a> { debug!(?target_bb, ?c.target, "register"); self.opportunities.push(ThreadingOpportunity { chain: vec![], target: c.target }); } - } else if target_bb == targets.otherwise() { - let (value, _, _) = targets.as_static_if()?; + } else if let Some((value, _, else_bb)) = targets.as_static_if() + && target_bb == else_bb + { let value = ScalarInt::try_from_uint(value, discr_layout.size)?; - // Likewise, we know that `discr != value`. That's a must weaker information, - // so we can only match the exact same condition. + // We only know that `discr != value`. That's much weaker information than + // the equality we had in the previous arm. All we can conclude is that + // the replacement condition `discr != value` can be threaded, and nothing else. for c in conditions.iter() { if c.value == value && c.polarity == Polarity::Ne { debug!(?target_bb, ?c.target, "register"); From 54b61d6e1a881b9da56df338ca674986d36f164f Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Thu, 19 Oct 2023 16:21:42 +0000 Subject: [PATCH 13/16] Reword TO application comments. --- compiler/rustc_mir_transform/src/jump_threading.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 19ba403ec6cbf..5a5ca6fa87019 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -639,7 +639,7 @@ impl OpportunitySet { let Some((current, chain)) = op_chain.split_first() else { return }; let basic_blocks = body.basic_blocks.as_mut(); - // Invariant: we never change the meaning of the program. + // Invariant: the control-flow is well-formed at the end of each iteration. let mut current = *current; for &succ in chain { debug!(?current, ?succ); @@ -697,8 +697,9 @@ impl OpportunitySet { new_involved.push((to_index, in_to_index + 1)); } } - // Following TOs new reference `new_succ`, so we will need to update them if we - // duplicate `new_succ` later. + + // The TOs that we just updated now reference `new_succ`. Update `involving_tos` + // in case we need to duplicate an edge starting at `new_succ` later. let _new_succ = self.involving_tos.push(new_involved); debug_assert_eq!(new_succ, _new_succ); From 8fb99afb0219846f98a82059de2f0808e4c0f61f Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Fri, 20 Oct 2023 15:24:09 +0000 Subject: [PATCH 14/16] Correct loop_headers logic. --- compiler/rustc_mir_transform/src/jump_threading.rs | 4 ++-- .../jump_threading.dfa.JumpThreading.panic-abort.diff | 8 +------- .../jump_threading.dfa.JumpThreading.panic-unwind.diff | 8 +------- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 5a5ca6fa87019..83af640f77726 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -743,7 +743,7 @@ enum Update { /// Compute the set of loop headers in the given body. We define a loop header as a block which has /// at least a predecessor which it dominates. This definition is only correct for reducible CFGs. /// But if the CFG is already irreducible, there is no point in trying much harder. -/// is already irreducibl +/// is already irreducible. fn loop_headers(body: &Body<'_>) -> BitSet { let mut loop_headers = BitSet::new_empty(body.basic_blocks.len()); let dominators = body.basic_blocks.dominators(); @@ -751,7 +751,7 @@ fn loop_headers(body: &Body<'_>) -> BitSet { for (bb, bbdata) in traversal::preorder(body) { for succ in bbdata.terminator().successors() { if dominators.dominates(succ, bb) { - loop_headers.insert(bb); + loop_headers.insert(succ); } } } diff --git a/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-abort.diff index 01bddc9a173f4..ad5846c97decc 100644 --- a/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-abort.diff @@ -19,8 +19,7 @@ StorageLive(_1); _1 = DFA::A; StorageLive(_2); -- goto -> bb1; -+ goto -> bb7; + goto -> bb1; } bb1: { @@ -64,11 +63,6 @@ _3 = const (); StorageDead(_7); goto -> bb1; -+ } -+ -+ bb7: { -+ _4 = discriminant(_1); -+ goto -> bb4; } } diff --git a/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-unwind.diff index 01bddc9a173f4..ad5846c97decc 100644 --- a/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.dfa.JumpThreading.panic-unwind.diff @@ -19,8 +19,7 @@ StorageLive(_1); _1 = DFA::A; StorageLive(_2); -- goto -> bb1; -+ goto -> bb7; + goto -> bb1; } bb1: { @@ -64,11 +63,6 @@ _3 = const (); StorageDead(_7); goto -> bb1; -+ } -+ -+ bb7: { -+ _4 = discriminant(_1); -+ goto -> bb4; } } From a845bac6ba434483b483c0561011227c1eff9daf Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Fri, 20 Oct 2023 15:50:09 +0000 Subject: [PATCH 15/16] FileCheck annotations. --- ...ppearing_bb.JumpThreading.panic-abort.diff | 11 +- ...pearing_bb.JumpThreading.panic-unwind.diff | 11 +- tests/mir-opt/jump_threading.rs | 229 ++++++++++++++++-- 3 files changed, 217 insertions(+), 34 deletions(-) diff --git a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff index 9c84aff536107..f290da84e5d88 100644 --- a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff +++ b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-abort.diff @@ -3,14 +3,11 @@ fn disappearing_bb(_1: u8) -> u8 { let mut _0: u8; - let mut _2: i8; + let mut _2: bool; let mut _3: bool; - let mut _4: bool; bb0: { - _4 = const false; - _3 = const false; - _4 = const true; + _2 = const true; _3 = const true; switchInt(_1) -> [0: bb3, 1: bb3, 2: bb1, otherwise: bb2]; } @@ -26,7 +23,7 @@ } bb3: { - _4 = const false; + _2 = const false; goto -> bb4; } @@ -35,7 +32,7 @@ } bb5: { - switchInt(_4) -> [0: bb6, otherwise: bb8]; + switchInt(_2) -> [0: bb6, otherwise: bb8]; } bb6: { diff --git a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff index 9c84aff536107..f290da84e5d88 100644 --- a/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff +++ b/tests/mir-opt/jump_threading.disappearing_bb.JumpThreading.panic-unwind.diff @@ -3,14 +3,11 @@ fn disappearing_bb(_1: u8) -> u8 { let mut _0: u8; - let mut _2: i8; + let mut _2: bool; let mut _3: bool; - let mut _4: bool; bb0: { - _4 = const false; - _3 = const false; - _4 = const true; + _2 = const true; _3 = const true; switchInt(_1) -> [0: bb3, 1: bb3, 2: bb1, otherwise: bb2]; } @@ -26,7 +23,7 @@ } bb3: { - _4 = const false; + _2 = const false; goto -> bb4; } @@ -35,7 +32,7 @@ } bb5: { - switchInt(_4) -> [0: bb6, otherwise: bb8]; + switchInt(_2) -> [0: bb6, otherwise: bb8]; } bb6: { diff --git a/tests/mir-opt/jump_threading.rs b/tests/mir-opt/jump_threading.rs index dcf6177356544..852dcd0db0112 100644 --- a/tests/mir-opt/jump_threading.rs +++ b/tests/mir-opt/jump_threading.rs @@ -1,5 +1,5 @@ // unit-test: JumpThreading -// compile-flags: -Zmir-enable-passes=+Inline -Zinline-mir +// compile-flags: -Zmir-enable-passes=+Inline // EMIT_MIR_FOR_EACH_PANIC_STRATEGY #![feature(control_flow_enum)] @@ -10,6 +10,31 @@ use std::intrinsics::mir::*; use std::ops::ControlFlow; fn too_complex(x: Result) -> Option { + // CHECK-LABEL: fn too_complex( + // CHECK: bb0: { + // CHECK: switchInt(move {{_.*}}) -> [0: bb3, 1: bb1, otherwise: bb2]; + // CHECK: bb1: { + // CHECK: [[controlflow:_.*]] = ControlFlow::::Break( + // CHECK: goto -> bb8; + // CHECK: bb2: { + // CHECK: unreachable; + // CHECK: bb3: { + // CHECK: [[controlflow]] = ControlFlow::::Continue( + // CHECK: goto -> bb4; + // CHECK: bb4: { + // CHECK: goto -> bb6; + // CHECK: bb5: { + // CHECK: {{_.*}} = (([[controlflow]] as Break).0: usize); + // CHECK: _0 = Option::::None; + // CHECK: goto -> bb7; + // CHECK: bb6: { + // CHECK: {{_.*}} = (([[controlflow]] as Continue).0: i32); + // CHECK: _0 = Option::::Some( + // CHECK: goto -> bb7; + // CHECK: bb7: { + // CHECK: return; + // CHECK: bb8: { + // CHECK: goto -> bb5; match { match x { Ok(v) => ControlFlow::Continue(v), @@ -22,6 +47,36 @@ fn too_complex(x: Result) -> Option { } fn identity(x: Result) -> Result { + // CHECK-LABEL: fn identity( + // CHECK: bb0: { + // CHECK: [[x:_.*]] = _1; + // CHECK: switchInt(move {{_.*}}) -> [0: bb8, 1: bb6, otherwise: bb7]; + // CHECK: bb1: { + // CHECK: goto -> bb2; + // CHECK: bb2: { + // CHECK: {{_.*}} = (([[controlflow:_.*]] as Continue).0: i32); + // CHECK: _0 = Result::::Ok( + // CHECK: goto -> bb5; + // CHECK: bb3: { + // CHECK: unreachable; + // CHECK: bb4: { + // CHECK: {{_.*}} = (([[controlflow]] as Break).0: std::result::Result); + // CHECK: _0 = Result::::Err( + // CHECK: goto -> bb5; + // CHECK: bb5: { + // CHECK: return; + // CHECK: bb6: { + // CHECK: {{_.*}} = move (([[x]] as Err).0: i32); + // CHECK: [[controlflow]] = ControlFlow::, i32>::Break( + // CHECK: goto -> bb9; + // CHECK: bb7: { + // CHECK: unreachable; + // CHECK: bb8: { + // CHECK: {{_.*}} = move (([[x]] as Ok).0: i32); + // CHECK: [[controlflow]] = ControlFlow::, i32>::Continue( + // CHECK: goto -> bb1; + // CHECK: bb9: { + // CHECK: goto -> bb4; Ok(x?) } @@ -32,7 +87,28 @@ enum DFA { D, } +/// Check that we do not thread through a loop header, +/// to avoid creating an irreducible CFG. fn dfa() { + // CHECK-LABEL: fn dfa( + // CHECK: bb0: { + // CHECK: {{_.*}} = DFA::A; + // CHECK: goto -> bb1; + // CHECK: bb1: { + // CHECK: switchInt({{.*}}) -> [0: bb4, 1: bb5, 2: bb6, 3: bb2, otherwise: bb3]; + // CHECK: bb2: { + // CHECK: return; + // CHECK: bb3: { + // CHECK: unreachable; + // CHECK: bb4: { + // CHECK: {{_.*}} = DFA::B; + // CHECK: goto -> bb1; + // CHECK: bb5: { + // CHECK: {{_.*}} = DFA::C; + // CHECK: goto -> bb1; + // CHECK: bb6: { + // CHECK: {{_.*}} = DFA::D; + // CHECK: goto -> bb1; let mut state = DFA::A; loop { match state { @@ -51,7 +127,29 @@ enum CustomDiscr { C = 99, } +/// Verify that we correctly match the discriminant value, and not its index. fn custom_discr(x: bool) -> u8 { + // CHECK-LABEL: fn custom_discr( + // CHECK: bb0: { + // CHECK: switchInt({{.*}}) -> [0: bb2, otherwise: bb1]; + // CHECK: bb1: { + // CHECK: {{_.*}} = CustomDiscr::A; + // CHECK: goto -> bb7; + // CHECK: bb2: { + // CHECK: {{_.*}} = CustomDiscr::B; + // CHECK: goto -> bb3; + // CHECK: bb3: { + // CHECK: goto -> bb4; + // CHECK: bb4: { + // CHECK: _0 = const 13_u8; + // CHECK: goto -> bb6; + // CHECK: bb5: { + // CHECK: _0 = const 5_u8; + // CHECK: goto -> bb6; + // CHECK: bb6: { + // CHECK: return; + // CHECK: bb7: { + // CHECK: goto -> bb5; match if x { CustomDiscr::A } else { CustomDiscr::B } { CustomDiscr::A => 5, _ => 13, @@ -60,76 +158,125 @@ fn custom_discr(x: bool) -> u8 { #[custom_mir(dialect = "runtime", phase = "post-cleanup")] fn multiple_match(x: u8) -> u8 { + // CHECK-LABEL: fn multiple_match( mir!( { + // CHECK: bb0: { + // CHECK: switchInt([[x:_.*]]) -> [3: bb1, otherwise: bb2]; match x { 3 => bb1, _ => bb2 } } bb1 = { // We know `x == 3`, so we can take `bb3`. + // CHECK: bb1: { + // CHECK: {{_.*}} = [[x]]; + // CHECK: goto -> bb3; let y = x; match y { 3 => bb3, _ => bb4 } } bb2 = { // We know `x != 3`, so we can take `bb6`. + // CHECK: bb2: { + // CHECK: [[z:_.*]] = [[x]]; + // CHECK: goto -> bb6; let z = x; match z { 3 => bb5, _ => bb6 } } bb3 = { + // CHECK: bb3: { + // CHECK: _0 = const 5_u8; + // CHECK: return; RET = 5; Return() } bb4 = { + // CHECK: bb4: { + // CHECK: _0 = const 7_u8; + // CHECK: return; RET = 7; Return() } bb5 = { + // CHECK: bb5: { + // CHECK: _0 = const 9_u8; + // CHECK: return; RET = 9; Return() } bb6 = { // We know `z != 3`, so we CANNOT take `bb7`. + // CHECK: bb6: { + // CHECK: switchInt([[z]]) -> [1: bb7, otherwise: bb8]; match z { 1 => bb7, _ => bb8 } } bb7 = { + // CHECK: bb7: { + // CHECK: _0 = const 9_u8; + // CHECK: return; RET = 9; Return() } bb8 = { + // CHECK: bb8: { + // CHECK: _0 = const 11_u8; + // CHECK: return; RET = 11; Return() } ) } +/// Both 1-3-4 and 2-3-4 are threadable. As 1 and 2 are the only predecessors of 3, +/// verify that we only thread the 3-4 part. #[custom_mir(dialect = "runtime", phase = "post-cleanup")] fn duplicate_chain(x: bool) -> u8 { + // CHECK-LABEL: fn duplicate_chain( mir!( let a: u8; { + // CHECK: bb0: { + // CHECK: switchInt({{.*}}) -> [1: bb1, otherwise: bb2]; match x { true => bb1, _ => bb2 } } bb1 = { + // CHECK: bb1: { + // CHECK: [[a:_.*]] = const 5_u8; + // CHECK: goto -> bb3; a = 5; Goto(bb3) } bb2 = { + // CHECK: bb2: { + // CHECK: [[a]] = const 5_u8; + // CHECK: goto -> bb3; a = 5; Goto(bb3) } - // Verify that we do not create multiple copied of `bb3`. bb3 = { + // CHECK: bb3: { + // CHECK: {{_.*}} = const 13_i32; + // CHECK: goto -> bb4; let b = 13; Goto(bb4) } bb4 = { + // CHECK: bb4: { + // CHECK: {{_.*}} = const 15_i32; + // CHECK-NOT: switchInt( + // CHECK: goto -> bb5; let c = 15; match a { 5 => bb5, _ => bb6 } } bb5 = { + // CHECK: bb5: { + // CHECK: _0 = const 7_u8; + // CHECK: return; RET = 7; Return() } bb6 = { + // CHECK: bb6: { + // CHECK: _0 = const 9_u8; + // CHECK: return; RET = 9; Return() } @@ -140,8 +287,13 @@ fn duplicate_chain(x: bool) -> u8 { #[rustc_nonnull_optimization_guaranteed] struct NonZeroUsize(usize); +/// Verify that we correctly discard threads that may mutate a discriminant by aliasing. #[custom_mir(dialect = "runtime", phase = "post-cleanup")] fn mutate_discriminant() -> u8 { + // CHECK-LABEL: fn mutate_discriminant( + // CHECK-NOT: goto -> {{bb.*}}; + // CHECK: switchInt( + // CHECK-NOT: goto -> {{bb.*}}; mir!( let x: Option; { @@ -166,8 +318,16 @@ fn mutate_discriminant() -> u8 { ) } -// Verify that we do not try to reason when there are mutable pointers involved. +/// Verify that we do not try to reason when there are mutable pointers involved. fn mutable_ref() -> bool { + // CHECK-LABEL: fn mutable_ref( + // CHECK-NOT: goto -> {{bb.*}}; + // CHECK: switchInt( + // CHECK: goto -> [[bbret:bb.*]]; + // CHECK: goto -> [[bbret]]; + // CHECK: [[bbret]]: { + // CHECK-NOT: {{bb.*}}: { + // CHECK: return; let mut x = 5; let a = std::ptr::addr_of_mut!(x); x = 7; @@ -179,80 +339,105 @@ fn mutable_ref() -> bool { } } +/// This function has 2 TOs: 1-3-4 and 0-1-3-4-6. +/// We verify that the second TO does not modify 3 once the first has been applied. #[custom_mir(dialect = "runtime", phase = "post-cleanup")] fn renumbered_bb(x: bool) -> u8 { - // This function has 2 TOs: 1-3-4 and 0-1-3-4-6. - // We verify that the second TO does not modify 3 once the first has been applied. + // CHECK-LABEL: fn renumbered_bb( mir!( let a: bool; let b: bool; { + // CHECK: bb0: { + // CHECK: switchInt({{.*}}) -> [1: bb1, otherwise: bb2]; b = false; match x { true => bb1, _ => bb2 } } bb1 = { + // CHECK: bb1: { + // CHECK: goto -> bb8; a = false; Goto(bb3) } bb2 = { + // CHECK: bb2: { + // CHECK: goto -> bb3; a = x; b = x; Goto(bb3) } bb3 = { + // CHECK: bb3: { + // CHECK: switchInt({{.*}}) -> [0: bb4, otherwise: bb5]; match a { false => bb4, _ => bb5 } } bb4 = { + // CHECK: bb4: { + // CHECK: switchInt({{.*}}) -> [0: bb6, otherwise: bb7]; match b { false => bb6, _ => bb7 } } bb5 = { + // CHECK: bb5: { + // CHECK: _0 = const 7_u8; RET = 7; Return() } bb6 = { + // CHECK: bb6: { + // CHECK: _0 = const 9_u8; RET = 9; Return() } bb7 = { + // CHECK: bb7: { + // CHECK: _0 = const 11_u8; RET = 11; Return() } + // Duplicate of bb3. + // CHECK: bb8: { + // CHECK-NEXT: goto -> bb9; + // Duplicate of bb4. + // CHECK: bb9: { + // CHECK-NEXT: goto -> bb6; ) } +/// This function has 3 TOs: 1-4-5, 0-1-4-7-5-8 and 3-4-7-5-6 +/// After applying the first TO, we create bb9 to replace 4, and rename 1-4 edge by 1-9. The +/// second TO may try to thread non-existing edge 9-4. +/// This test verifies that we preserve semantics by bailing out of this second TO. #[custom_mir(dialect = "runtime", phase = "post-cleanup")] fn disappearing_bb(x: u8) -> u8 { - // This function has 3 TOs: 1-4-5, 0-1-4-7-5-8 and 3-4-7-5-6 - // After applying the first TO, we create bb9 to replace 4, and rename 1-4 edge by 1-9. The - // second TO may try to thread non-existing edge 9-4. - // This test verifies that we preserve semantics by bailing out of this second TO. + // CHECK-LABEL: fn disappearing_bb( mir!( - let _11: i8; - let _12: bool; - let _13: bool; + let a: bool; + let b: bool; { - _13 = false; - _12 = false; - _13 = true; - _12 = true; + a = true; + b = true; match x { 0 => bb3, 1 => bb3, 2 => bb1, _ => bb2 } } bb1 = { - _12 = false; + // CHECK: bb1: { + // CHECK: goto -> bb9; + b = false; Goto(bb4) } bb2 = { Unreachable() } bb3 = { - _13 = false; + // CHECK: bb3: { + // CHECK: goto -> bb10; + a = false; Goto(bb4) } bb4 = { - match _12 { false => bb5, _ => bb7 } + match b { false => bb5, _ => bb7 } } bb5 = { - match _13 { false => bb6, _ => bb8 } + match a { false => bb6, _ => bb8 } } bb6 = { Return() @@ -263,6 +448,10 @@ fn disappearing_bb(x: u8) -> u8 { bb8 = { Goto(bb6) } + // CHECK: bb9: { + // CHECK: goto -> bb5; + // CHECK: bb10: { + // CHECK: goto -> bb6; ) } From dd08dd42e52af9677c3fcf83477d9693883cb221 Mon Sep 17 00:00:00 2001 From: Camille GILLOT Date: Fri, 20 Oct 2023 15:51:46 +0000 Subject: [PATCH 16/16] Expand comment on disappearing edge. --- compiler/rustc_mir_transform/src/jump_threading.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/rustc_mir_transform/src/jump_threading.rs b/compiler/rustc_mir_transform/src/jump_threading.rs index 83af640f77726..7b918be447413 100644 --- a/compiler/rustc_mir_transform/src/jump_threading.rs +++ b/compiler/rustc_mir_transform/src/jump_threading.rs @@ -645,7 +645,7 @@ impl OpportunitySet { debug!(?current, ?succ); // `succ` must be a successor of `current`. If it is not, this means this TO is not - // satisfiable, so we bail out. + // satisfiable and a previous TO erased this edge, so we bail out. if basic_blocks[current].terminator().successors().find(|s| *s == succ).is_none() { debug!("impossible"); return;