From 9f837addf5c6446d05502299ab719d48a4117170 Mon Sep 17 00:00:00 2001 From: Eduard-Mihai Burtescu Date: Thu, 1 Apr 2021 08:30:22 +0300 Subject: [PATCH 1/3] structurizer: split regions into `Divergent` vs `Convergent`. --- crates/rustc_codegen_spirv/src/lib.rs | 1 + .../src/linker/new_structurizer.rs | 332 +++++++++++------- 2 files changed, 206 insertions(+), 127 deletions(-) diff --git a/crates/rustc_codegen_spirv/src/lib.rs b/crates/rustc_codegen_spirv/src/lib.rs index b6912da4a7..f54596d034 100644 --- a/crates/rustc_codegen_spirv/src/lib.rs +++ b/crates/rustc_codegen_spirv/src/lib.rs @@ -16,6 +16,7 @@ //! [`spirv-tools`]: https://embarkstudios.github.io/rust-gpu/api/spirv_tools //! [`spirv-tools-sys`]: https://embarkstudios.github.io/rust-gpu/api/spirv_tools_sys #![feature(rustc_private)] +#![feature(assert_matches)] #![feature(once_cell)] // BEGIN - Embark standard lints v0.3 // do not change or add/remove here, but one can add exceptions after this section diff --git a/crates/rustc_codegen_spirv/src/linker/new_structurizer.rs b/crates/rustc_codegen_spirv/src/linker/new_structurizer.rs index d36a2b175c..40e9b5aa1b 100644 --- a/crates/rustc_codegen_spirv/src/linker/new_structurizer.rs +++ b/crates/rustc_codegen_spirv/src/linker/new_structurizer.rs @@ -114,19 +114,42 @@ type BlockId = Word; /// Regions are made up of their entry block and all other blocks dominated /// by that block. All edges leaving a region are considered "exits". -struct Region { - /// After structurizing a region, all paths through it must lead to a single - /// "merge" block (i.e. `merge` post-dominates the entire region). - /// The `merge` block must be terminated by one of `OpReturn`, `OpReturnValue`, - /// `OpKill`, or `OpUnreachable`. If `exits` isn't empty, `merge` will - /// receive an `OpBranch` from its parent region (to an outer merge block). +#[derive(Debug)] +enum Region { + /// All paths through the region "diverge", i.e. they never leave the region + /// by branching to another block in the same function, so they either: + /// * get stuck in an (infinite) loop + /// * reach `OpReturn`, `OpReturnValue`, `OpKill`, or `OpUnreachable` + /// + /// Such a region is fully structurized and requires no propagation to its + /// parent region (only that a branch into its entry block exists). + Divergent, + + /// Some paths through the region don't "diverge" (see `ConvergentRegion`). + Convergent(ConvergentRegion), +} + +/// `Region` where at least some paths through the region leave it by branching +/// to other blocks in the same function. +/// +/// After structurizing, all paths through it must lead to a single "merge" block +/// (i.e. `merge` post-dominates the entire region), with all of the branches to +/// outside the region having been moved from the CFG itself, into `exits`. +/// +/// The `merge` block is terminated by `OpUnreachable`, to be replaced with an +/// `OpBranch` (to an outer merge block) by its parent region, which will also +/// inherit its `exits` (potentially attaching some/all of them to itself). +#[derive(Debug)] +struct ConvergentRegion { merge: BlockIdx, merge_id: BlockId, + // FIXME(eddyb) use something more compact when `exits.len()` is small + // (e.g. `ArrayVec<[(BlockIdx, Exit); 4]>` with linear search). exits: IndexMap, } -#[derive(Default)] +#[derive(Default, Debug)] struct Exit { /// Number of total edges to this target (a subset of the target's predecessors). edge_count: usize, @@ -168,11 +191,7 @@ impl Structurizer<'_> { let block_id = self.func.blocks()[block].label_id().unwrap(); let terminator = self.func.blocks()[block].instructions.last().unwrap(); let mut region = match terminator.class.opcode { - Op::Return | Op::ReturnValue | Op::Kill | Op::Unreachable => Region { - merge: block, - merge_id: block_id, - exits: indexmap! {}, - }, + Op::Return | Op::ReturnValue | Op::Kill | Op::Unreachable => Region::Divergent, Op::Branch => { let target = self.block_id_to_idx[&terminator.operands[0].unwrap_id_ref()]; @@ -181,13 +200,13 @@ impl Structurizer<'_> { self.func.builder.pop_instruction().unwrap(); // Default all merges to `OpUnreachable`, in case they're unused. self.func.builder.unreachable().unwrap(); - Region { + Region::Convergent(ConvergentRegion { merge: block, merge_id: block_id, exits: indexmap! { target => Exit { edge_count: 1, condition: None } }, - } + }) }) } @@ -199,7 +218,7 @@ impl Structurizer<'_> { }; // FIXME(eddyb) avoid wasteful allocation. - let child_regions: Vec<_> = target_operand_indices + let convergent_child_regions: Vec<_> = target_operand_indices .map(|i| { let target_id = self.func.blocks()[block] .instructions @@ -223,84 +242,134 @@ impl Structurizer<'_> { .last_mut() .unwrap() .operands[i] = Operand::IdRef(new_block_id); - Region { + Region::Convergent(ConvergentRegion { merge: new_block, merge_id: new_block_id, exits: indexmap! { target => Exit { edge_count: 1, condition: None } }, - } + }) }) }) + .filter_map(|region| match region { + Region::Divergent => None, + Region::Convergent(region) => Some(region), + }) .collect(); - self.selection_merge_regions(block, &child_regions) + self.selection_merge_convergent_regions(block, &convergent_child_regions) } _ => panic!("Invalid block terminator: {:?}", terminator), }; // Peel off deferred exits which have all their edges accounted for // already, within this region. Repeat until no such exits are left. - while let Some((&target, _)) = region - .exits - .iter() - .find(|&(&target, exit)| exit.edge_count == self.incoming_edge_count[target]) - { - let taken_block_id = self.func.blocks()[target].label_id().unwrap(); - let exit = region.exits.remove(&target).unwrap(); - - // Special-case the last exit as unconditional - regardless of - // what might end up in `exit.condition`, what we'd generate is - // `if exit.condition { branch target; } else { unreachable; }` - // which is just `branch target;` with an extra assumption that - // `exit.condition` is `true` (which we can just ignore). - if region.exits.is_empty() { - self.func.builder.select_block(Some(region.merge)).unwrap(); - assert_eq!( - self.func.builder.pop_instruction().unwrap().class.opcode, - Op::Unreachable - ); - self.func.builder.branch(taken_block_id).unwrap(); - region = self.regions.remove(&target).unwrap(); - continue; - } + while let Region::Convergent(convergent_region) = &mut region { + let (region_merge, target, exit) = + match convergent_region.exits.iter().find(|&(&target, exit)| { + exit.edge_count == self.incoming_edge_count[target] + }) { + Some((&target, _)) => { + let exit = convergent_region.exits.remove(&target).unwrap(); + let region_merge = convergent_region.merge; + if convergent_region.exits.is_empty() { + region = Region::Divergent; + } + (region_merge, target, exit) + } + None => { + break; + } + }; - // Create a new block for the "`exit` not taken" path. - let not_taken_block_id = self.func.builder.begin_block(None).unwrap(); - let not_taken_block = self.func.builder.selected_block().unwrap(); - // Default all merges to `OpUnreachable`, in case they're unused. - self.func.builder.unreachable().unwrap(); + let taken_block_id = self.func.blocks()[target].label_id().unwrap(); + let taken_region = self.regions.remove(&target).unwrap(); // Choose whether to take this `exit`, in the previous merge block. - let branch_block = region.merge; - self.func.builder.select_block(Some(branch_block)).unwrap(); - assert_eq!( - self.func.builder.pop_instruction().unwrap().class.opcode, - Op::Unreachable - ); - self.func - .builder - .branch_conditional( - exit.condition.unwrap(), - taken_block_id, - not_taken_block_id, - iter::empty(), - ) - .unwrap(); + region = match region { + Region::Divergent => { + // Special-case the last exit as unconditional - regardless of + // what might end up in `exit.condition`, what we'd generate is + // `if exit.condition { branch target; } else { unreachable; }` + // which is just `branch target;` with an extra assumption that + // `exit.condition` is `true` (which we can just ignore). + self.func.builder.select_block(Some(region_merge)).unwrap(); + assert_eq!( + self.func.builder.pop_instruction().unwrap().class.opcode, + Op::Unreachable + ); + self.func.builder.branch(taken_block_id).unwrap(); + taken_region + } + Region::Convergent(ConvergentRegion { exits, .. }) => { + // Create a new block for the "`exit` not taken" path. + let not_taken_block_id = self.func.builder.begin_block(None).unwrap(); + let not_taken_block = self.func.builder.selected_block().unwrap(); + // Default all merges to `OpUnreachable`, in case they're unused. + self.func.builder.unreachable().unwrap(); - // Merge the "taken" and "not taken" paths. - let taken_region = self.regions.remove(&target).unwrap(); - let not_taken_region = Region { - merge: not_taken_block, - merge_id: not_taken_block_id, - exits: region.exits, + let not_taken_region = ConvergentRegion { + merge: not_taken_block, + merge_id: not_taken_block_id, + exits, + }; + + self.func.builder.select_block(Some(region_merge)).unwrap(); + assert_eq!( + self.func.builder.pop_instruction().unwrap().class.opcode, + Op::Unreachable + ); + self.func + .builder + .branch_conditional( + exit.condition.unwrap(), + taken_block_id, + not_taken_block_id, + iter::empty(), + ) + .unwrap(); + + // Merge the "taken" and "not taken" paths. + match taken_region { + Region::Divergent => { + self.func.builder.select_block(Some(region_merge)).unwrap(); + self.func + .builder + .insert_selection_merge( + InsertPoint::FromEnd(1), + not_taken_block_id, + SelectionControl::NONE, + ) + .unwrap(); + Region::Convergent(not_taken_region) + } + Region::Convergent(taken_region) => self + .selection_merge_convergent_regions( + region_merge, + &[taken_region, not_taken_region], + ), + } + } }; - region = - self.selection_merge_regions(branch_block, &[taken_region, not_taken_region]); } // Peel off a backedge exit, which indicates this region is a loop. - if let Some(mut backedge_exit) = region.exits.remove(&block) { + let region_merge_and_backedge_exit = match &mut region { + Region::Divergent => None, + Region::Convergent(convergent_region) => { + match convergent_region.exits.remove(&block) { + Some(backedge_exit) => { + let region_merge = convergent_region.merge; + if convergent_region.exits.is_empty() { + region = Region::Divergent; + } + Some((region_merge, backedge_exit)) + } + None => None, + } + } + }; + if let Some((mut region_merge, mut backedge_exit)) = region_merge_and_backedge_exit { // Inject a `while`-like loop header just before the start of the // loop body. This is needed because our "`break` vs `continue`" // choice is *after* the loop body, like in a `do`-`while` loop, @@ -322,9 +391,8 @@ impl Structurizer<'_> { // we just moved (this should only be relevant to infinite loops). self.func.blocks_mut()[while_body_block].instructions = mem::replace(&mut self.func.blocks_mut()[block].instructions, vec![]); - if region.merge == block { - region.merge = while_body_block; - region.merge_id = while_body_block_id; + if region_merge == block { + region_merge = while_body_block; } // Create a separate merge block for the loop body, as the original @@ -332,7 +400,7 @@ impl Structurizer<'_> { let while_body_merge_id = self.func.builder.begin_block(None).unwrap(); let while_body_merge = self.func.builder.selected_block().unwrap(); self.func.builder.select_block(None).unwrap(); - self.func.builder.select_block(Some(region.merge)).unwrap(); + self.func.builder.select_block(Some(region_merge)).unwrap(); assert_eq!( self.func.builder.pop_instruction().unwrap().class.opcode, Op::Unreachable @@ -354,9 +422,13 @@ impl Structurizer<'_> { .select_block(Some(while_header_block)) .unwrap(); - for (&target, exit) in region - .exits - .iter_mut() + let region_exits = match &mut region { + Region::Divergent => None, + Region::Convergent(ConvergentRegion { exits, .. }) => Some(exits.iter_mut()), + }; + for (&target, exit) in region_exits + .into_iter() + .flatten() .chain(iter::once((&while_body_block, &mut backedge_exit))) { let first_entry_case = ( @@ -404,8 +476,16 @@ impl Structurizer<'_> { iter::empty(), ) .unwrap(); - region.merge = while_exit_block; - region.merge_id = while_exit_block_id; + region = match region { + Region::Divergent => Region::Divergent, + Region::Convergent(ConvergentRegion { exits, .. }) => { + Region::Convergent(ConvergentRegion { + merge: while_exit_block, + merge_id: while_exit_block_id, + exits, + }) + } + }; // Remove the backedge count from the total incoming count of `block`. // This will allow outer regions to treat the loop opaquely. @@ -416,7 +496,7 @@ impl Structurizer<'_> { } assert_eq!(self.regions.len(), 1); - assert_eq!(self.regions.values().next().unwrap().exits.len(), 0); + assert_matches!(self.regions.values().next().unwrap(), Region::Divergent); } fn child_region(&mut self, target: BlockIdx) -> Option { @@ -428,7 +508,11 @@ impl Structurizer<'_> { } } - fn selection_merge_regions(&mut self, block: BlockIdx, child_regions: &[Region]) -> Region { + fn selection_merge_convergent_regions( + &mut self, + block: BlockIdx, + child_regions: &[ConvergentRegion], + ) -> Region { let Globals { const_false, const_true, @@ -437,27 +521,28 @@ impl Structurizer<'_> { // HACK(eddyb) this special-cases the easy case where we can // just reuse a merge block, and don't have to create our own. - let unconditional_single_exit = |region: &Region| { + let unconditional_single_exit = |region: &ConvergentRegion| { region.exits.len() == 1 && region.exits.get_index(0).unwrap().1.condition.is_none() }; - let structural_merge = if child_regions.iter().all(unconditional_single_exit) { - let merge = *child_regions[0].exits.get_index(0).unwrap().0; - if child_regions - .iter() - .all(|region| *region.exits.get_index(0).unwrap().0 == merge) - && child_regions + let structural_merge = + if !child_regions.is_empty() && child_regions.iter().all(unconditional_single_exit) { + let merge = *child_regions[0].exits.get_index(0).unwrap().0; + if child_regions .iter() - .map(|region| region.exits.get_index(0).unwrap().1.edge_count) - .sum::() - == self.incoming_edge_count[merge] - { - Some(merge) + .all(|region| *region.exits.get_index(0).unwrap().0 == merge) + && child_regions + .iter() + .map(|region| region.exits.get_index(0).unwrap().1.edge_count) + .sum::() + == self.incoming_edge_count[merge] + { + Some(merge) + } else { + None + } } else { None - } - } else { - None - }; + }; // Reuse or create a merge block, and use it as the selection merge. let merge = structural_merge.unwrap_or_else(|| { @@ -473,16 +558,12 @@ impl Structurizer<'_> { // Branch all the child regions into our merge block. for region in child_regions { - // HACK(eddyb) empty `region.exits` indicate diverging control-flow, - // and that we should ignore `region.merge`. - if !region.exits.is_empty() { - self.func.builder.select_block(Some(region.merge)).unwrap(); - assert_eq!( - self.func.builder.pop_instruction().unwrap().class.opcode, - Op::Unreachable - ); - self.func.builder.branch(merge_id).unwrap(); - } + self.func.builder.select_block(Some(region.merge)).unwrap(); + assert_eq!( + self.func.builder.pop_instruction().unwrap().class.opcode, + Op::Unreachable + ); + self.func.builder.branch(merge_id).unwrap(); } if let Some(merge) = structural_merge { @@ -500,32 +581,29 @@ impl Structurizer<'_> { // Update conditions using phis. for (&target, exit) in &mut exits { - let phi_cases = child_regions - .iter() - .filter(|region| { - // HACK(eddyb) empty `region.exits` indicate diverging control-flow, - // and that we should ignore `region.merge`. - !region.exits.is_empty() - }) - .map(|region| { - ( - match region.exits.get(&target) { - Some(exit) => exit.condition.unwrap_or(const_true), - None => const_false, - }, - region.merge_id, - ) - }); + let phi_cases = child_regions.iter().map(|region| { + ( + match region.exits.get(&target) { + Some(exit) => exit.condition.unwrap_or(const_true), + None => const_false, + }, + region.merge_id, + ) + }); exit.condition = Some(self.func.builder.phi(type_bool, None, phi_cases).unwrap()); } // Default all merges to `OpUnreachable`, in case they're unused. self.func.builder.unreachable().unwrap(); - Region { - merge, - merge_id, - exits, + if exits.is_empty() { + Region::Divergent + } else { + Region::Convergent(ConvergentRegion { + merge, + merge_id, + exits, + }) } } } From ec5498a1a3e56a50e9576f32d93035cbb7ee118d Mon Sep 17 00:00:00 2001 From: Eduard-Mihai Burtescu Date: Thu, 1 Apr 2021 09:15:14 +0300 Subject: [PATCH 2/3] structurizer: simplify `OpPhi`s where all values are the same. --- .../src/linker/new_structurizer.rs | 26 +++++++--- crates/spirv-builder/src/test/basic.rs | 47 +++++++++---------- 2 files changed, 41 insertions(+), 32 deletions(-) diff --git a/crates/rustc_codegen_spirv/src/linker/new_structurizer.rs b/crates/rustc_codegen_spirv/src/linker/new_structurizer.rs index 40e9b5aa1b..98593b51fc 100644 --- a/crates/rustc_codegen_spirv/src/linker/new_structurizer.rs +++ b/crates/rustc_codegen_spirv/src/linker/new_structurizer.rs @@ -441,12 +441,8 @@ impl Structurizer<'_> { ); let repeat_case = (exit.condition.unwrap_or(const_true), while_body_merge_id); let phi_cases = [first_entry_case, repeat_case]; - exit.condition = Some( - self.func - .builder - .phi(type_bool, None, phi_cases.iter().copied()) - .unwrap(), - ); + exit.condition = + Some(self.emit_phi_or_simplfy(type_bool, phi_cases.iter().copied())); } // Choose whether to keep looping, in the `while`-like loop header. @@ -590,7 +586,7 @@ impl Structurizer<'_> { region.merge_id, ) }); - exit.condition = Some(self.func.builder.phi(type_bool, None, phi_cases).unwrap()); + exit.condition = Some(self.emit_phi_or_simplfy(type_bool, phi_cases)); } // Default all merges to `OpUnreachable`, in case they're unused. @@ -608,6 +604,22 @@ impl Structurizer<'_> { } } + /// Emit an `OpPhi`, but only if it can't be simplified, such as when all + /// values are the same (and therefore the common value itself can be used). + fn emit_phi_or_simplfy( + &mut self, + value_type: Word, + cases: impl Iterator + Clone, + ) -> Word { + let mut values = cases.clone().map(|(v, _)| v); + let first = values.next().unwrap(); + if values.all(|other| other == first) { + first + } else { + self.func.builder.phi(value_type, None, cases).unwrap() + } + } + // FIXME(eddyb) replace this with `rustc_data_structures::graph::iterate` // (or similar). fn post_order(&mut self) -> Vec { diff --git a/crates/spirv-builder/src/test/basic.rs b/crates/spirv-builder/src/test/basic.rs index 1963be7f45..8f1fda0093 100644 --- a/crates/spirv-builder/src/test/basic.rs +++ b/crates/spirv-builder/src/test/basic.rs @@ -233,28 +233,26 @@ OpBranch %8 %8 = OpLabel %9 = OpPhi %10 %11 %7 %12 %13 %14 = OpPhi %2 %4 %7 %15 %13 -%16 = OpPhi %17 %18 %7 %19 %13 -OpLoopMerge %20 %13 Unroll -OpBranchConditional %16 %21 %20 -%21 = OpLabel -%22 = OpSLessThan %17 %9 %23 -OpSelectionMerge %24 None -OpBranchConditional %22 %25 %26 -%25 = OpLabel -%27 = OpIMul %2 %28 %14 -%29 = OpIAdd %2 %27 %5 -%30 = OpIAdd %10 %9 %31 -OpBranch %24 -%26 = OpLabel -OpReturnValue %14 +OpLoopMerge %16 %13 Unroll +OpBranchConditional %17 %18 %16 +%18 = OpLabel +%19 = OpSLessThan %20 %9 %21 +OpSelectionMerge %22 None +OpBranchConditional %19 %23 %24 +%23 = OpLabel +%25 = OpIMul %2 %26 %14 +%27 = OpIAdd %2 %25 %5 +%28 = OpIAdd %10 %9 %29 +OpBranch %22 %24 = OpLabel -%12 = OpPhi %10 %30 %25 -%15 = OpPhi %2 %29 %25 -%19 = OpPhi %17 %18 %25 +OpReturnValue %14 +%22 = OpLabel +%12 = OpPhi %10 %28 %23 +%15 = OpPhi %2 %27 %23 OpBranch %13 %13 = OpLabel OpBranch %8 -%20 = OpLabel +%16 = OpLabel OpUnreachable OpFunctionEnd"#, ); @@ -493,14 +491,13 @@ OpBranch %25 %25 = OpLabel OpBranch %26 %26 = OpLabel -%27 = OpPhi %16 %28 %25 %28 %29 -OpLoopMerge %30 %29 None -OpBranchConditional %27 %31 %30 -%31 = OpLabel -OpBranch %29 -%29 = OpLabel -OpBranch %26 +OpLoopMerge %27 %28 None +OpBranchConditional %29 %30 %27 %30 = OpLabel +OpBranch %28 +%28 = OpLabel +OpBranch %26 +%27 = OpLabel OpUnreachable %17 = OpLabel OpUnreachable From 509e5f21aa2290e80dc91182396ab9b6a42ed377 Mon Sep 17 00:00:00 2001 From: Eduard-Mihai Burtescu Date: Thu, 1 Apr 2021 10:03:06 +0300 Subject: [PATCH 3/3] structurizer: remove unnecessary empty blocks on "(exit) not taken" paths. --- .../src/linker/new_structurizer.rs | 68 ++++++++----------- 1 file changed, 29 insertions(+), 39 deletions(-) diff --git a/crates/rustc_codegen_spirv/src/linker/new_structurizer.rs b/crates/rustc_codegen_spirv/src/linker/new_structurizer.rs index 98593b51fc..ce5aabe4c3 100644 --- a/crates/rustc_codegen_spirv/src/linker/new_structurizer.rs +++ b/crates/rustc_codegen_spirv/src/linker/new_structurizer.rs @@ -286,7 +286,7 @@ impl Structurizer<'_> { let taken_region = self.regions.remove(&target).unwrap(); // Choose whether to take this `exit`, in the previous merge block. - region = match region { + match region { Region::Divergent => { // Special-case the last exit as unconditional - regardless of // what might end up in `exit.condition`, what we'd generate is @@ -299,26 +299,37 @@ impl Structurizer<'_> { Op::Unreachable ); self.func.builder.branch(taken_block_id).unwrap(); - taken_region + region = taken_region; } - Region::Convergent(ConvergentRegion { exits, .. }) => { - // Create a new block for the "`exit` not taken" path. - let not_taken_block_id = self.func.builder.begin_block(None).unwrap(); - let not_taken_block = self.func.builder.selected_block().unwrap(); - // Default all merges to `OpUnreachable`, in case they're unused. - self.func.builder.unreachable().unwrap(); - - let not_taken_region = ConvergentRegion { - merge: not_taken_block, - merge_id: not_taken_block_id, - exits, + Region::Convergent(convergent_region) => { + // HACK(eddyb) since `selection_merge_convergent_regions` + // creates a fresh merge block, use that block for the + // "(exit) not taken" side of our conditional branch, + // instead of creating an additional empty block for it. + // However, note that this trick requires us merging the + // region itself with the "(exit) taken" edge, and then + // rewriting the unconditional branch from `region_merge` + // to the "(exit) not taken" block, into a conditional one. + region = match taken_region { + Region::Divergent => self.selection_merge_convergent_regions( + region_merge, + &[convergent_region], + ), + Region::Convergent(taken_region) => self + .selection_merge_convergent_regions( + region_merge, + &[taken_region, convergent_region], + ), }; + // Rewrite the injected `OpBranch %not_taken` into + // `OpBranchConditional %exit_condition %taken %not_taken`. self.func.builder.select_block(Some(region_merge)).unwrap(); - assert_eq!( - self.func.builder.pop_instruction().unwrap().class.opcode, - Op::Unreachable - ); + let not_taken_block_id = { + let branch_inst = self.func.builder.pop_instruction().unwrap(); + assert_eq!(branch_inst.class.opcode, Op::Branch); + branch_inst.operands[0].unwrap_id_ref() + }; self.func .builder .branch_conditional( @@ -328,29 +339,8 @@ impl Structurizer<'_> { iter::empty(), ) .unwrap(); - - // Merge the "taken" and "not taken" paths. - match taken_region { - Region::Divergent => { - self.func.builder.select_block(Some(region_merge)).unwrap(); - self.func - .builder - .insert_selection_merge( - InsertPoint::FromEnd(1), - not_taken_block_id, - SelectionControl::NONE, - ) - .unwrap(); - Region::Convergent(not_taken_region) - } - Region::Convergent(taken_region) => self - .selection_merge_convergent_regions( - region_merge, - &[taken_region, not_taken_region], - ), - } } - }; + } } // Peel off a backedge exit, which indicates this region is a loop.