diff --git a/compiler/rustc_middle/src/mir/basic_blocks.rs b/compiler/rustc_middle/src/mir/basic_blocks.rs index 0d2e23609ce35..f4df9d289f8c6 100644 --- a/compiler/rustc_middle/src/mir/basic_blocks.rs +++ b/compiler/rustc_middle/src/mir/basic_blocks.rs @@ -33,6 +33,93 @@ struct Cache { predecessors: OnceLock, reverse_postorder: OnceLock>, dominators: OnceLock>, + is_cyclic: OnceLock, + sccs: OnceLock, +} + +#[derive(Clone, Default, Debug)] +pub struct SccData { + pub component_count: usize, + + /// The SCC of each block. + pub components: IndexVec, + + /// The contents of each SCC: its blocks, in RPO. + pub sccs: Vec>, +} + +use std::collections::VecDeque; + +struct PearceRecursive { + r_index: IndexVec, + stack: VecDeque, + index: u32, + c: u32, +} + +impl PearceRecursive { + fn new(node_count: usize) -> Self { + assert!(node_count > 0); // only a non-empty graph is supported + // todo: assert node_count is within overflow limits + Self { + r_index: IndexVec::from_elem_n(0, node_count), + stack: VecDeque::new(), + index: 1, + c: node_count.try_into().unwrap(), + // c: node_count - 1, + } + } + + fn compute_sccs(&mut self, blocks: &IndexVec>) { + for v in blocks.indices() { + if self.r_index[v] == 0 { + self.visit(v, blocks); + } + } + + // The SCC labels are from N - 1 to zero, remap them from 0 to the component count, to match + // their position in an array of SCCs. + let node_count: u32 = blocks.len().try_into().unwrap(); + for scc_index in self.r_index.iter_mut() { + *scc_index = node_count - *scc_index - 1; + } + + // Adjust the component index counter to the component count + self.c = node_count - self.c; + } + + fn visit(&mut self, v: BasicBlock, blocks: &IndexVec>) { + let mut root = true; + self.r_index[v] = self.index; + self.index += 1; + + for w in blocks[v].terminator().successors() { + if self.r_index[w] == 0 { + self.visit(w, blocks); + } + if self.r_index[w] < self.r_index[v] { + self.r_index[v] = self.r_index[w]; + root = false; + } + } + + if root { + self.index -= 1; + self.c -= 1; + + while let Some(&w) = self.stack.front() + && self.r_index[v] <= self.r_index[w] + { + self.stack.pop_front(); + self.r_index[w] = self.c; + self.index -= 1; + } + + self.r_index[v] = self.c; + } else { + self.stack.push_front(v); + } + } } impl<'tcx> BasicBlocks<'tcx> { @@ -41,10 +128,35 @@ impl<'tcx> BasicBlocks<'tcx> { BasicBlocks { basic_blocks, cache: Cache::default() } } + /// Returns true if control-flow graph contains a cycle reachable from the `START_BLOCK`. + #[inline] + pub fn is_cfg_cyclic(&self) -> bool { + *self.cache.is_cyclic.get_or_init(|| graph::is_cyclic(self)) + } + + #[inline] pub fn dominators(&self) -> &Dominators { self.cache.dominators.get_or_init(|| dominators(self)) } + #[inline] + pub fn sccs(&self) -> &SccData { + self.cache.sccs.get_or_init(|| { + let block_count = self.basic_blocks.len(); + + let mut pearce = PearceRecursive::new(block_count); + pearce.compute_sccs(&self.basic_blocks); + let component_count = pearce.c as usize; + + let mut sccs = vec![smallvec::SmallVec::new(); component_count]; + for &block in self.reverse_postorder().iter() { + let scc = pearce.r_index[block] as usize; + sccs[scc].push(block); + } + SccData { component_count, components: pearce.r_index, sccs } + }) + } + /// Returns predecessors for each basic block. #[inline] pub fn predecessors(&self) -> &Predecessors { diff --git a/compiler/rustc_mir_dataflow/src/framework/mod.rs b/compiler/rustc_mir_dataflow/src/framework/mod.rs index b6a5603601959..75b7076e03708 100644 --- a/compiler/rustc_mir_dataflow/src/framework/mod.rs +++ b/compiler/rustc_mir_dataflow/src/framework/mod.rs @@ -229,6 +229,27 @@ pub trait Analysis<'tcx> { unreachable!(); } + #[inline] + fn iterate_to_fixpoint<'mir>( + self, + tcx: TyCtxt<'tcx>, + body: &'mir mir::Body<'tcx>, + pass_name: Option<&'static str>, + ) -> AnalysisAndResults<'tcx, Self> + where + Self: Sized, + Self::Domain: DebugWithContext, + { + // Computing dataflow over the SCCs is only supported in forward analyses. It's also + // unnecessary to use it on acyclic graphs, as the condensation graph is of course the same + // as the CFG itself. + if Self::Direction::IS_BACKWARD || !body.basic_blocks.is_cfg_cyclic() { + self.iterate_to_fixpoint_per_block(tcx, body, pass_name) + } else { + self.iterate_to_fixpoint_per_scc(tcx, body, pass_name) + } + } + /* Extension methods */ /// Finds the fixpoint for this dataflow problem. @@ -244,7 +265,7 @@ pub trait Analysis<'tcx> { /// dataflow analysis. Some analyses are run multiple times in the compilation pipeline. /// Without a `pass_name` to differentiates them, only the results for the latest run will be /// saved. - fn iterate_to_fixpoint<'mir>( + fn iterate_to_fixpoint_per_block<'mir>( mut self, tcx: TyCtxt<'tcx>, body: &'mir mir::Body<'tcx>, @@ -308,6 +329,92 @@ pub trait Analysis<'tcx> { AnalysisAndResults { analysis: self, results } } + + fn iterate_to_fixpoint_per_scc<'mir>( + mut self, + _tcx: TyCtxt<'tcx>, + body: &'mir mir::Body<'tcx>, + _pass_name: Option<&'static str>, + ) -> AnalysisAndResults<'tcx, Self> + where + Self: Sized, + Self::Domain: DebugWithContext, + { + assert!(Self::Direction::IS_FORWARD); + + let sccs = body.basic_blocks.sccs(); + + struct VecQueue { + queue: Vec, + set: DenseBitSet, + } + + impl VecQueue { + #[inline] + fn with_none(len: usize) -> Self { + VecQueue { queue: Vec::with_capacity(len), set: DenseBitSet::new_empty(len) } + } + + #[inline] + fn insert(&mut self, element: T) { + if self.set.insert(element) { + self.queue.push(element); + } + } + } + + let mut scc_queue = VecQueue::with_none(sccs.component_count); + for &bb in body.basic_blocks.reverse_postorder().iter() { + // let scc = sccs.components[bb.as_usize()]; + let scc = sccs.components[bb]; + scc_queue.insert(scc); + } + // assert_eq!(scc_queue.queue, sccs.queue); + + let mut results = IndexVec::from_fn_n(|_| self.bottom_value(body), body.basic_blocks.len()); + self.initialize_start_block(body, &mut results[mir::START_BLOCK]); + + // Worklist for per-SCC iterations + let mut dirty_queue: WorkQueue = WorkQueue::with_none(body.basic_blocks.len()); + + let mut state = self.bottom_value(body); + + for &scc in &scc_queue.queue { + // les blocks doivent être ajoutés en RPO + // for block in sccs.blocks_in_rpo(scc as usize) { + for block in sccs.sccs[scc as usize].iter().copied() { + dirty_queue.insert(block); + } + + while let Some(bb) = dirty_queue.pop() { + // Set the state to the entry state of the block. This is equivalent to `state = + // results[bb].clone()`, but it saves an allocation, thus improving compile times. + state.clone_from(&results[bb]); + + Self::Direction::apply_effects_in_block( + &mut self, + body, + &mut state, + bb, + &body[bb], + |target: BasicBlock, state: &Self::Domain| { + let set_changed = results[target].join(state); + // let target_scc = sccs.components[target.as_usize()]; + let target_scc = sccs.components[target]; + if set_changed && target_scc == scc { + // The target block is in the SCC we're currently processing, and we + // want to process this block until fixpoint. Otherwise, the target + // block is in a successor SCC and it will be processed when that SCC is + // encountered later. + dirty_queue.insert(target); + } + }, + ); + } + } + + AnalysisAndResults { analysis: self, results } + } } /// The legal operations for a transfer function in a gen/kill problem.