diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 3315d022..8c96ee11 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -49,6 +49,12 @@ path = "fuzz_targets/ion_checker.rs" test = false doc = false +[[bin]] +name = "fastalloc_checker" +path = "fuzz_targets/fastalloc_checker.rs" +test = false +doc = false + # Enable debug assertions and overflow checks when fuzzing [profile.release] debug = true diff --git a/fuzz/fuzz_targets/fastalloc_checker.rs b/fuzz/fuzz_targets/fastalloc_checker.rs new file mode 100644 index 00000000..b099c27a --- /dev/null +++ b/fuzz/fuzz_targets/fastalloc_checker.rs @@ -0,0 +1,45 @@ +/* + * Released under the terms of the Apache 2.0 license with LLVM + * exception. See `LICENSE` for details. + */ + +#![no_main] +use regalloc2::fuzzing::arbitrary::{Arbitrary, Result, Unstructured}; +use regalloc2::fuzzing::checker::Checker; +use regalloc2::fuzzing::func::{Func, Options}; +use regalloc2::fuzzing::fuzz_target; + +#[derive(Clone, Debug)] +struct TestCase { + func: Func, +} + +impl Arbitrary<'_> for TestCase { + fn arbitrary(u: &mut Unstructured) -> Result { + Ok(TestCase { + func: Func::arbitrary_with_options( + u, + &Options { + reused_inputs: true, + fixed_regs: true, + fixed_nonallocatable: true, + clobbers: true, + reftypes: false, + }, + )?, + }) + } +} + +fuzz_target!(|testcase: TestCase| { + let func = testcase.func; + let _ = env_logger::try_init(); + log::trace!("func:\n{:?}", func); + let env = regalloc2::fuzzing::func::machine_env(); + let out = + regalloc2::fuzzing::fastalloc::run(&func, &env, true, false).expect("regalloc did not succeed"); + + let mut checker = Checker::new(&func, &env); + checker.prepare(&out); + checker.run().expect("checker failed"); +}); diff --git a/regalloc2-tool/src/main.rs b/regalloc2-tool/src/main.rs index 5a03d754..b5a21842 100644 --- a/regalloc2-tool/src/main.rs +++ b/regalloc2-tool/src/main.rs @@ -2,8 +2,8 @@ use std::path::PathBuf; use clap::Parser; use regalloc2::{ - checker::Checker, serialize::SerializableFunction, Block, Edit, Function, InstOrEdit, Output, - RegallocOptions, + checker::Checker, serialize::SerializableFunction, Algorithm, Block, Edit, Function, + InstOrEdit, Output, RegallocOptions, }; #[derive(Parser)] @@ -15,6 +15,24 @@ struct Args { /// Input file containing a bincode-encoded SerializedFunction. input: PathBuf, + + /// Which register allocation algorithm to use. + algorithm: CliAlgorithm, +} + +#[derive(Clone, Copy, Debug, clap::ValueEnum)] +enum CliAlgorithm { + Ion, + Fastalloc, +} + +impl From for Algorithm { + fn from(cli_algo: CliAlgorithm) -> Algorithm { + match cli_algo { + CliAlgorithm::Ion => Algorithm::Ion, + CliAlgorithm::Fastalloc => Algorithm::Fastalloc, + } + } } fn main() { @@ -32,6 +50,7 @@ fn main() { let options = RegallocOptions { verbose_log: true, validate_ssa: true, + algorithm: args.algorithm.into(), }; let output = match regalloc2::run(&function, function.machine_env(), &options) { Ok(output) => output, diff --git a/src/fastalloc/iter.rs b/src/fastalloc/iter.rs new file mode 100644 index 00000000..d1437559 --- /dev/null +++ b/src/fastalloc/iter.rs @@ -0,0 +1,44 @@ +use crate::{Operand, OperandConstraint, OperandKind}; + +pub struct Operands<'a>(pub &'a [Operand]); + +impl<'a> Operands<'a> { + pub fn new(operands: &'a [Operand]) -> Self { + Self(operands) + } + + pub fn matches bool + 'a>( + &self, + predicate: F, + ) -> impl Iterator + 'a { + self.0 + .iter() + .cloned() + .enumerate() + .filter(move |(_, op)| predicate(*op)) + } + + pub fn def_ops(&self) -> impl Iterator + 'a { + self.matches(|op| op.kind() == OperandKind::Def) + } + + pub fn use_ops(&self) -> impl Iterator + 'a { + self.matches(|op| op.kind() == OperandKind::Use) + } + + pub fn reuse(&self) -> impl Iterator + 'a { + self.matches(|op| matches!(op.constraint(), OperandConstraint::Reuse(_))) + } + + pub fn fixed(&self) -> impl Iterator + 'a { + self.matches(|op| matches!(op.constraint(), OperandConstraint::FixedReg(_))) + } +} + +impl<'a> core::ops::Index for Operands<'a> { + type Output = Operand; + + fn index(&self, index: usize) -> &Self::Output { + &self.0[index] + } +} diff --git a/src/fastalloc/lru.rs b/src/fastalloc/lru.rs new file mode 100644 index 00000000..37c61155 --- /dev/null +++ b/src/fastalloc/lru.rs @@ -0,0 +1,324 @@ +use crate::{FxHashSet, PReg, PRegSet, RegClass}; +use alloc::vec; +use alloc::vec::Vec; +use core::{ + fmt, + ops::{Index, IndexMut}, +}; + +/// A least-recently-used cache organized as a linked list based on a vector. +pub struct Lru { + /// The list of node information. + /// + /// Each node corresponds to a physical register. + /// The index of a node is the `address` from the perspective of the linked list. + pub data: Vec, + /// Index of the most recently used register. + pub head: u8, + /// Class of registers in the cache. + pub regclass: RegClass, +} + +#[derive(Clone, Copy, Debug)] +pub struct LruNode { + /// The previous physical register in the list. + pub prev: u8, + /// The next physical register in the list. + pub next: u8, +} + +impl Lru { + pub fn new(regclass: RegClass, regs: &[PReg]) -> Self { + let mut data = vec![ + LruNode { + prev: u8::MAX, + next: u8::MAX + }; + PReg::MAX + 1 + ]; + let no_of_regs = regs.len(); + for i in 0..no_of_regs { + let (reg, prev_reg, next_reg) = ( + regs[i], + regs[i.checked_sub(1).unwrap_or(no_of_regs - 1)], + regs[if i >= no_of_regs - 1 { 0 } else { i + 1 }], + ); + data[reg.hw_enc()].prev = prev_reg.hw_enc() as u8; + data[reg.hw_enc()].next = next_reg.hw_enc() as u8; + } + Self { + head: if regs.is_empty() { + u8::MAX + } else { + regs[0].hw_enc() as u8 + }, + data, + regclass, + } + } + + /// Marks the physical register `preg` as the most recently used + pub fn poke(&mut self, preg: PReg) { + trace!( + "Before poking: {:?} LRU. head: {:?}, Actual data: {:?}", + self.regclass, + self.head, + self.data + ); + trace!("About to poke {:?} in {:?} LRU", preg, self.regclass); + let prev_newest = self.head; + let hw_enc = preg.hw_enc() as u8; + if hw_enc == prev_newest { + return; + } + if self.data[prev_newest as usize].prev != hw_enc { + self.remove(hw_enc as usize); + self.insert_before(hw_enc, self.head); + } + self.head = hw_enc; + trace!("Poked {:?} in {:?} LRU", preg, self.regclass); + if cfg!(debug_assertions) { + self.validate_lru(); + } + } + + /// Gets the least recently used physical register. + pub fn pop(&mut self) -> PReg { + trace!( + "Before popping: {:?} LRU. head: {:?}, Actual data: {:?}", + self.regclass, + self.head, + self.data + ); + trace!("Popping {:?} LRU", self.regclass); + if self.is_empty() { + panic!("LRU is empty"); + } + let oldest = self.data[self.head as usize].prev; + trace!("Popped p{oldest} in {:?} LRU", self.regclass); + if cfg!(debug_assertions) { + self.validate_lru(); + } + PReg::new(oldest as usize, self.regclass) + } + + /// Get the last PReg in the LRU from the set `from`. + pub fn last(&self, from: PRegSet) -> Option { + trace!("Getting the last preg from the LRU in set {from}"); + self.last_satisfying(|preg| from.contains(preg)) + } + + /// Get the last PReg from the LRU for which `f` returns true. + pub fn last_satisfying bool>(&self, f: F) -> Option { + trace!("Getting the last preg from the LRU satisfying..."); + if self.is_empty() { + panic!("LRU is empty"); + } + let mut last = self.data[self.head as usize].prev; + let init_last = last; + loop { + let preg = PReg::new(last as usize, self.regclass); + if f(preg) { + return Some(preg); + } + last = self.data[last as usize].prev; + if last == init_last { + return None; + } + } + } + + /// Splices out a node from the list. + fn remove(&mut self, hw_enc: usize) { + trace!( + "Before removing: {:?} LRU. head: {:?}, Actual data: {:?}", + self.regclass, + self.head, + self.data + ); + trace!("Removing p{hw_enc} from {:?} LRU", self.regclass); + let (iprev, inext) = ( + self.data[hw_enc].prev as usize, + self.data[hw_enc].next as usize, + ); + self.data[iprev].next = self.data[hw_enc].next; + self.data[inext].prev = self.data[hw_enc].prev; + self.data[hw_enc].prev = u8::MAX; + self.data[hw_enc].next = u8::MAX; + if hw_enc == self.head as usize { + if hw_enc == inext { + // There are no regs in the LRU + self.head = u8::MAX; + } else { + self.head = inext as u8; + } + } + trace!("Removed p{hw_enc} from {:?} LRU", self.regclass); + if cfg!(debug_assertions) { + self.validate_lru(); + } + } + + /// Insert node `i` before node `j` in the list. + fn insert_before(&mut self, i: u8, j: u8) { + trace!( + "Before inserting: {:?} LRU. head: {:?}, Actual data: {:?}", + self.regclass, + self.head, + self.data + ); + trace!("Inserting p{i} before {j} in {:?} LRU", self.regclass); + let prev = self.data[j as usize].prev; + self.data[prev as usize].next = i; + self.data[j as usize].prev = i; + self.data[i as usize] = LruNode { next: j, prev }; + trace!("Done inserting p{i} before {j} in {:?} LRU", self.regclass); + if cfg!(debug_assertions) { + self.validate_lru(); + } + } + + pub fn is_empty(&self) -> bool { + self.head == u8::MAX + } + + // Using this to debug. + fn validate_lru(&self) { + trace!( + "{:?} LRU. head: {:?}, Actual data: {:?}", + self.regclass, + self.head, + self.data + ); + if self.head != u8::MAX { + let mut node = self.data[self.head as usize].next; + let mut seen = FxHashSet::default(); + while node != self.head { + if seen.contains(&node) { + panic!( + "Cycle detected in {:?} LRU.\n + head: {:?}, actual data: {:?}", + self.regclass, self.head, self.data + ); + } + seen.insert(node); + node = self.data[node as usize].next; + } + for i in 0..self.data.len() { + if self.data[i].prev == u8::MAX && self.data[i].next == u8::MAX { + // Removed + continue; + } + if self.data[i].prev == u8::MAX || self.data[i].next == u8::MAX { + panic!( + "Invalid LRU. p{} next or previous is an invalid value, but not both", + i + ); + } + if self.data[self.data[i].prev as usize].next != i as u8 { + panic!( + "Invalid LRU. p{i} prev is p{:?}, but p{:?} next is {:?}", + self.data[i].prev, + self.data[i].prev, + self.data[self.data[i].prev as usize].next + ); + } + if self.data[self.data[i].next as usize].prev != i as u8 { + panic!( + "Invalid LRU. p{i} next is p{:?}, but p{:?} prev is p{:?}", + self.data[i].next, + self.data[i].next, + self.data[self.data[i].next as usize].prev + ); + } + } + } + } +} + +impl fmt::Debug for Lru { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use alloc::format; + let data_str = if self.head == u8::MAX { + format!("") + } else { + let mut data_str = format!("p{}", self.head); + let mut node = self.data[self.head as usize].next; + let mut seen = FxHashSet::default(); + while node != self.head { + if seen.contains(&node) { + panic!( + "The {:?} LRU is messed up: + head: {:?}, {:?} -> p{node}, actual data: {:?}", + self.regclass, self.head, data_str, self.data + ); + } + seen.insert(node); + data_str += &format!(" -> p{}", node); + node = self.data[node as usize].next; + } + data_str + }; + f.debug_struct("Lru") + .field("head", if self.is_empty() { &"none" } else { &self.head }) + .field("class", &self.regclass) + .field("data", &data_str) + .finish() + } +} + +#[derive(Clone)] +pub struct PartedByRegClass { + pub items: [T; 3], +} + +impl Index for PartedByRegClass { + type Output = T; + + fn index(&self, index: RegClass) -> &Self::Output { + &self.items[index as usize] + } +} + +impl IndexMut for PartedByRegClass { + fn index_mut(&mut self, index: RegClass) -> &mut Self::Output { + &mut self.items[index as usize] + } +} + +/// Least-recently-used caches for register classes Int, Float, and Vector, respectively. +pub type Lrus = PartedByRegClass; + +impl Lrus { + pub fn new(int_regs: &[PReg], float_regs: &[PReg], vec_regs: &[PReg]) -> Self { + Self { + items: [ + Lru::new(RegClass::Int, int_regs), + Lru::new(RegClass::Float, float_regs), + Lru::new(RegClass::Vector, vec_regs), + ], + } + } +} + +use core::fmt::{Debug, Display}; + +impl Display for PartedByRegClass { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{{ int: {}, float: {}, vector: {} }}", + self.items[0], self.items[1], self.items[2] + ) + } +} + +impl Debug for PartedByRegClass { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{{ int: {:?}, float: {:?}, vector: {:?} }}", + self.items[0], self.items[1], self.items[2] + ) + } +} diff --git a/src/fastalloc/mod.rs b/src/fastalloc/mod.rs new file mode 100644 index 00000000..49680158 --- /dev/null +++ b/src/fastalloc/mod.rs @@ -0,0 +1,1314 @@ +use crate::moves::{MoveAndScratchResolver, ParallelMoves}; +use crate::{cfg::CFGInfo, ion::Stats, Allocation, RegAllocError}; +use crate::{ssa::validate_ssa, Edit, Function, MachineEnv, Output, ProgPoint}; +use crate::{ + AllocationKind, Block, FxHashMap, Inst, InstPosition, Operand, OperandConstraint, OperandKind, + OperandPos, PReg, PRegSet, RegClass, SpillSlot, VReg, +}; +use alloc::vec::Vec; +use core::convert::TryInto; +use core::iter::FromIterator; +use core::ops::{Index, IndexMut}; + +mod iter; +mod lru; +mod vregset; +use iter::*; +use lru::*; +use vregset::VRegSet; + +#[cfg(test)] +mod tests; + +#[derive(Debug)] +struct Allocs { + allocs: Vec, + /// `inst_alloc_offsets[i]` is the offset into `allocs` for the allocations of + /// instruction `i`'s operands. + inst_alloc_offsets: Vec, +} + +impl Allocs { + fn new(func: &F) -> (Self, u32) { + let mut allocs = Vec::new(); + let mut inst_alloc_offsets = Vec::with_capacity(func.num_insts()); + let mut max_operand_len = 0; + let mut no_of_operands = 0; + for inst in 0..func.num_insts() { + let operands_len = func.inst_operands(Inst::new(inst)).len() as u32; + max_operand_len = max_operand_len.max(operands_len); + inst_alloc_offsets.push(no_of_operands as u32); + no_of_operands += operands_len; + } + allocs.resize(no_of_operands as usize, Allocation::none()); + ( + Self { + allocs, + inst_alloc_offsets, + }, + max_operand_len, + ) + } +} + +impl Index<(usize, usize)> for Allocs { + type Output = Allocation; + + /// Retrieve the allocation for operand `idx.1` at instruction `idx.0` + fn index(&self, idx: (usize, usize)) -> &Allocation { + &self.allocs[self.inst_alloc_offsets[idx.0] as usize + idx.1] + } +} + +impl IndexMut<(usize, usize)> for Allocs { + fn index_mut(&mut self, idx: (usize, usize)) -> &mut Allocation { + &mut self.allocs[self.inst_alloc_offsets[idx.0] as usize + idx.1] + } +} + +#[derive(Debug)] +struct Stack<'a, F: Function> { + num_spillslots: u32, + func: &'a F, +} + +impl<'a, F: Function> Stack<'a, F> { + fn new(func: &'a F) -> Self { + Self { + num_spillslots: 0, + func, + } + } + + /// Allocates a spill slot on the stack for `vreg` + fn allocstack(&mut self, class: RegClass) -> SpillSlot { + trace!("Allocating a spillslot for class {class:?}"); + let size: u32 = self.func.spillslot_size(class).try_into().unwrap(); + // Rest of this function was copied verbatim + // from `Env::allocate_spillslot` in src/ion/spill.rs. + let mut offset = self.num_spillslots; + // Align up to `size`. + debug_assert!(size.is_power_of_two()); + offset = (offset + size - 1) & !(size - 1); + let slot = if self.func.multi_spillslot_named_by_last_slot() { + offset + size - 1 + } else { + offset + }; + offset += size; + self.num_spillslots = offset; + trace!("Allocated slot: {slot}"); + SpillSlot::new(slot as usize) + } +} + +#[derive(Debug)] +struct Edits { + /// The final output edits. + edits: Vec<(ProgPoint, Edit)>, + fixed_stack_slots: PRegSet, + /// The scratch registers being used in the instruction being + /// currently processed. + scratch_regs: PartedByRegClass>, + dedicated_scratch_regs: PartedByRegClass>, +} + +impl Edits { + fn new( + fixed_stack_slots: PRegSet, + num_insts: usize, + dedicated_scratch_regs: PartedByRegClass>, + ) -> Self { + // This guess is based on the sightglass benchmarks: + // The average number of edits per instruction is 1. + let edits_len_guess = num_insts; + Self { + edits: Vec::with_capacity(edits_len_guess), + fixed_stack_slots, + scratch_regs: dedicated_scratch_regs.clone(), + dedicated_scratch_regs, + } + } +} + +impl Edits { + fn is_stack(&self, alloc: Allocation) -> bool { + alloc.is_stack() + || (alloc.is_reg() && self.fixed_stack_slots.contains(alloc.as_reg().unwrap())) + } + + fn add_move( + &mut self, + inst: Inst, + from: Allocation, + to: Allocation, + class: RegClass, + pos: InstPosition, + ) { + trace!( + "Recording edit: {:?}", + (ProgPoint::new(inst, pos), Edit::Move { from, to }, class) + ); + if self.is_stack(from) && self.is_stack(to) { + trace!("Edit is stack-to-stack. Generating two edits with a scratch register"); + let scratch_reg = self.scratch_regs[class].unwrap(); + let scratch_alloc = Allocation::reg(scratch_reg); + trace!("Move 1: {scratch_alloc:?} to {to:?}"); + self.edits.push(( + ProgPoint::new(inst, pos), + Edit::Move { + from: scratch_alloc, + to, + }, + )); + trace!("Move 2: {from:?} to {scratch_alloc:?}"); + self.edits.push(( + ProgPoint::new(inst, pos), + Edit::Move { + from, + to: scratch_alloc, + }, + )); + } else { + self.edits + .push((ProgPoint::new(inst, pos), Edit::Move { from, to })); + } + } +} + +#[derive(Debug, Clone)] +struct PartedByOperandPos { + items: [T; 2], +} + +impl Index for PartedByOperandPos { + type Output = T; + fn index(&self, index: OperandPos) -> &Self::Output { + &self.items[index as usize] + } +} + +impl IndexMut for PartedByOperandPos { + fn index_mut(&mut self, index: OperandPos) -> &mut Self::Output { + &mut self.items[index as usize] + } +} + +use core::fmt; + +impl fmt::Display for PartedByOperandPos { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{{ early: {}, late: {} }}", self.items[0], self.items[1]) + } +} + +#[derive(Debug)] +pub struct Env<'a, F: Function> { + func: &'a F, + + /// The current allocations for all virtual registers. + vreg_allocs: Vec, + /// Spillslots for all virtual registers. + /// `vreg_spillslots[i]` is the spillslot for virtual register `i`. + vreg_spillslots: Vec, + /// The virtual registers that are currently live. + live_vregs: VRegSet, + /// Least-recently-used caches for register classes Int, Float, and Vector, respectively. + lrus: Lrus, + /// `vreg_in_preg[i]` is the virtual register currently in the physical register + /// with index `i`. + vreg_in_preg: Vec, + /// `reused_input_to_reuse_op[i]` is the operand index of the reuse operand + /// that uses the `i`th operand in the current instruction as its input. + reused_input_to_reuse_op: Vec, + /// The set of registers that can be used for allocation in the + /// early and late phases of an instruction. + /// Allocatable registers that contain no vregs, registers that can be + /// evicted can be in the set, and fixed stack slots are in this set. + available_pregs: PartedByOperandPos, + init_available_pregs: PRegSet, + allocatable_regs: PRegSet, + stack: Stack<'a, F>, + preferred_victim: PartedByRegClass, + vreg_to_live_inst_range: Vec<(ProgPoint, ProgPoint, Allocation)>, + + fixed_stack_slots: PRegSet, + + // Output. + allocs: Allocs, + edits: Edits, + stats: Stats, + debug_locations: Vec<(u32, ProgPoint, ProgPoint, Allocation)>, +} + +impl<'a, F: Function> Env<'a, F> { + fn new(func: &'a F, env: &'a MachineEnv) -> Self { + use alloc::vec; + let mut regs = [ + env.preferred_regs_by_class[RegClass::Int as usize].clone(), + env.preferred_regs_by_class[RegClass::Float as usize].clone(), + env.preferred_regs_by_class[RegClass::Vector as usize].clone(), + ]; + regs[0].extend( + env.non_preferred_regs_by_class[RegClass::Int as usize] + .iter() + .cloned(), + ); + regs[1].extend( + env.non_preferred_regs_by_class[RegClass::Float as usize] + .iter() + .cloned(), + ); + regs[2].extend( + env.non_preferred_regs_by_class[RegClass::Vector as usize] + .iter() + .cloned(), + ); + let allocatable_regs = PRegSet::from(env); + let init_available_pregs = { + let mut regs = allocatable_regs; + for preg in env.fixed_stack_slots.iter() { + regs.add(*preg); + } + regs + }; + let dedicated_scratch_regs = PartedByRegClass { + items: [ + env.scratch_by_class[0], + env.scratch_by_class[1], + env.scratch_by_class[2], + ], + }; + trace!("{:?}", env); + let (allocs, max_operand_len) = Allocs::new(func); + let fixed_stack_slots = PRegSet::from_iter(env.fixed_stack_slots.iter().cloned()); + Self { + func, + allocatable_regs, + vreg_allocs: vec![Allocation::none(); func.num_vregs()], + vreg_spillslots: vec![SpillSlot::invalid(); func.num_vregs()], + live_vregs: VRegSet::with_capacity(func.num_vregs()), + lrus: Lrus::new(®s[0], ®s[1], ®s[2]), + vreg_in_preg: vec![VReg::invalid(); PReg::NUM_INDEX], + stack: Stack::new(func), + fixed_stack_slots, + vreg_to_live_inst_range: vec![ + ( + ProgPoint::invalid(), + ProgPoint::invalid(), + Allocation::none() + ); + func.num_vregs() + ], + preferred_victim: PartedByRegClass { + items: [ + regs[0].last().cloned().unwrap_or(PReg::invalid()), + regs[1].last().cloned().unwrap_or(PReg::invalid()), + regs[2].last().cloned().unwrap_or(PReg::invalid()), + ], + }, + reused_input_to_reuse_op: vec![usize::MAX; max_operand_len as usize], + init_available_pregs, + available_pregs: PartedByOperandPos { + items: [init_available_pregs, init_available_pregs], + }, + allocs, + edits: Edits::new(fixed_stack_slots, func.num_insts(), dedicated_scratch_regs), + stats: Stats::default(), + debug_locations: Vec::with_capacity(func.debug_value_labels().len()), + } + } + + fn reset_available_pregs_and_scratch_regs(&mut self) { + trace!("Resetting the available pregs"); + self.available_pregs = PartedByOperandPos { + items: [self.init_available_pregs, self.init_available_pregs], + }; + self.edits.scratch_regs = self.edits.dedicated_scratch_regs.clone(); + } + + fn alloc_scratch_reg(&mut self, inst: Inst, class: RegClass) -> Result<(), RegAllocError> { + use OperandPos::{Early, Late}; + let reg = self.get_scratch_reg( + inst, + class, + self.available_pregs[Late] & self.available_pregs[Early], + )?; + self.edits.scratch_regs[class] = Some(reg); + self.available_pregs[OperandPos::Early].remove(reg); + self.available_pregs[OperandPos::Late].remove(reg); + Ok(()) + } + + fn get_scratch_reg_for_reload( + &mut self, + inst: Inst, + class: RegClass, + avail_regs: PRegSet, + ) -> Result { + let Some(preg) = self.lrus[class].last(avail_regs) else { + return Err(RegAllocError::TooManyLiveRegs); + }; + if self.vreg_in_preg[preg.index()] != VReg::invalid() { + self.evict_vreg_in_preg_before_inst(inst, preg); + } + Ok(preg) + } + + fn get_scratch_reg( + &mut self, + inst: Inst, + class: RegClass, + avail_regs: PRegSet, + ) -> Result { + let Some(preg) = self.lrus[class].last(avail_regs) else { + return Err(RegAllocError::TooManyLiveRegs); + }; + if self.vreg_in_preg[preg.index()] != VReg::invalid() { + self.evict_vreg_in_preg(inst, preg); + } + Ok(preg) + } + + fn reserve_reg_for_fixed_operand( + &mut self, + op: Operand, + op_idx: usize, + preg: PReg, + ) -> Result<(), RegAllocError> { + trace!("Reserving register {preg} for fixed operand {op}"); + let early_avail_pregs = self.available_pregs[OperandPos::Early]; + let late_avail_pregs = self.available_pregs[OperandPos::Late]; + match (op.pos(), op.kind()) { + (OperandPos::Early, OperandKind::Use) => { + if op.as_fixed_nonallocatable().is_none() && !early_avail_pregs.contains(preg) { + return Err(RegAllocError::TooManyLiveRegs); + } + self.available_pregs[OperandPos::Early].remove(preg); + if self.reused_input_to_reuse_op[op_idx] != usize::MAX { + if op.as_fixed_nonallocatable().is_none() && !late_avail_pregs.contains(preg) { + return Err(RegAllocError::TooManyLiveRegs); + } + self.available_pregs[OperandPos::Late].remove(preg); + } + } + (OperandPos::Late, OperandKind::Def) => { + if op.as_fixed_nonallocatable().is_none() && !late_avail_pregs.contains(preg) { + return Err(RegAllocError::TooManyLiveRegs); + } + self.available_pregs[OperandPos::Late].remove(preg); + } + _ => { + if op.as_fixed_nonallocatable().is_none() + && (!early_avail_pregs.contains(preg) || !late_avail_pregs.contains(preg)) + { + return Err(RegAllocError::TooManyLiveRegs); + } + self.available_pregs[OperandPos::Early].remove(preg); + self.available_pregs[OperandPos::Late].remove(preg); + } + } + Ok(()) + } + + fn allocd_within_constraint(&self, op: Operand) -> bool { + let alloc = self.vreg_allocs[op.vreg().vreg()]; + match op.constraint() { + OperandConstraint::Any => { + if let Some(preg) = alloc.as_reg() { + if !self.available_pregs[op.pos()].contains(preg) { + // If a register isn't in the available pregs list, then + // there are two cases: either it's reserved for a + // fixed register constraint or a vreg allocated in the instruction + // is already assigned to it. + // + // For example: + // 1. use v0, use v0, use v0 + // + // Say p0 is assigned to v0 during the processing of the first operand. + // When the second v0 operand is being processed, v0 will still be in + // v0, so it is still allocated within constraints. + trace!("The vreg in {preg}: {}", self.vreg_in_preg[preg.index()]); + self.vreg_in_preg[preg.index()] == op.vreg() + } else { + true + } + } else { + !alloc.is_none() + } + } + OperandConstraint::Reg => { + if self.edits.is_stack(alloc) { + return false; + } + if let Some(preg) = alloc.as_reg() { + if !self.available_pregs[op.pos()].contains(preg) { + trace!("The vreg in {preg}: {}", self.vreg_in_preg[preg.index()]); + self.vreg_in_preg[preg.index()] == op.vreg() + } else { + true + } + } else { + false + } + } + // It is possible for an operand to have a fixed register constraint to + // a clobber. + OperandConstraint::FixedReg(preg) => alloc.is_reg() && alloc.as_reg().unwrap() == preg, + OperandConstraint::Reuse(_) => { + unreachable!() + } + } + } + + fn base_evict_vreg_in_preg(&mut self, inst: Inst, preg: PReg, pos: InstPosition) { + trace!("Removing the vreg in preg {} for eviction", preg); + let evicted_vreg = self.vreg_in_preg[preg.index()]; + trace!("The removed vreg: {}", evicted_vreg); + debug_assert_ne!(evicted_vreg, VReg::invalid()); + if self.vreg_spillslots[evicted_vreg.vreg()].is_invalid() { + self.vreg_spillslots[evicted_vreg.vreg()] = self.stack.allocstack(evicted_vreg.class()); + } + let slot = self.vreg_spillslots[evicted_vreg.vreg()]; + self.vreg_allocs[evicted_vreg.vreg()] = Allocation::stack(slot); + trace!("Move reason: eviction"); + self.edits.add_move( + inst, + self.vreg_allocs[evicted_vreg.vreg()], + Allocation::reg(preg), + evicted_vreg.class(), + pos, + ); + } + + fn evict_vreg_in_preg_before_inst(&mut self, inst: Inst, preg: PReg) { + self.base_evict_vreg_in_preg(inst, preg, InstPosition::Before) + } + + fn evict_vreg_in_preg(&mut self, inst: Inst, preg: PReg) { + self.base_evict_vreg_in_preg(inst, preg, InstPosition::After) + } + + fn freealloc(&mut self, vreg: VReg) { + trace!("Freeing vreg {}", vreg); + let alloc = self.vreg_allocs[vreg.vreg()]; + match alloc.kind() { + AllocationKind::Reg => { + let preg = alloc.as_reg().unwrap(); + self.vreg_in_preg[preg.index()] = VReg::invalid(); + } + AllocationKind::Stack => (), + AllocationKind::None => unreachable!("Attempting to free an unallocated operand!"), + } + self.vreg_allocs[vreg.vreg()] = Allocation::none(); + self.live_vregs.remove(vreg.vreg()); + trace!( + "{} curr alloc is now {}", + vreg, + self.vreg_allocs[vreg.vreg()] + ); + } + + /// Allocates a physical register for the operand `op`. + fn alloc_reg_for_operand( + &mut self, + inst: Inst, + op: Operand, + ) -> Result { + trace!("available regs: {}", self.available_pregs); + trace!("Int LRU: {:?}", self.lrus[RegClass::Int]); + trace!("Float LRU: {:?}", self.lrus[RegClass::Float]); + trace!("Vector LRU: {:?}", self.lrus[RegClass::Vector]); + trace!(""); + let draw_from = match (op.pos(), op.kind()) { + (OperandPos::Late, OperandKind::Use) + | (OperandPos::Early, OperandKind::Def) + | (OperandPos::Late, OperandKind::Def) + if matches!(op.constraint(), OperandConstraint::Reuse(_)) => + { + self.available_pregs[OperandPos::Late] & self.available_pregs[OperandPos::Early] + } + _ => self.available_pregs[op.pos()], + }; + if draw_from.is_empty(op.class()) { + trace!("No registers available for {op}"); + return Err(RegAllocError::TooManyLiveRegs); + } + let Some(preg) = self.lrus[op.class()].last(draw_from) else { + trace!( + "Failed to find an available {:?} register in the LRU for operand {op}", + op.class() + ); + return Err(RegAllocError::TooManyLiveRegs); + }; + if self.vreg_in_preg[preg.index()] != VReg::invalid() { + self.evict_vreg_in_preg(inst, preg); + } + trace!("The allocated register for vreg {}: {}", op.vreg(), preg); + self.lrus[op.class()].poke(preg); + self.available_pregs[op.pos()].remove(preg); + match (op.pos(), op.kind()) { + (OperandPos::Late, OperandKind::Use) => { + self.available_pregs[OperandPos::Early].remove(preg); + } + (OperandPos::Early, OperandKind::Def) => { + self.available_pregs[OperandPos::Late].remove(preg); + } + (OperandPos::Late, OperandKind::Def) + if matches!(op.constraint(), OperandConstraint::Reuse(_)) => + { + self.available_pregs[OperandPos::Early].remove(preg); + } + _ => (), + }; + Ok(Allocation::reg(preg)) + } + + /// Allocates for the operand `op` with index `op_idx` into the + /// vector of instruction `inst`'s operands. + fn alloc_operand( + &mut self, + inst: Inst, + op: Operand, + op_idx: usize, + ) -> Result { + let new_alloc = match op.constraint() { + OperandConstraint::Any => self.alloc_reg_for_operand(inst, op)?, + OperandConstraint::Reg => self.alloc_reg_for_operand(inst, op)?, + OperandConstraint::FixedReg(preg) => { + trace!("The fixed preg: {} for operand {}", preg, op); + + Allocation::reg(preg) + } + OperandConstraint::Reuse(_) => { + // This is handled elsewhere. + unreachable!(); + } + }; + self.allocs[(inst.index(), op_idx)] = new_alloc; + Ok(new_alloc) + } + + /// Allocate operand the `op_idx`th operand `op` in instruction `inst` within its constraint. + /// Since only fixed register constraints are allowed, `fixed_spillslot` is used when a + /// fixed stack allocation is needed, like when transferring a stack allocation from a + /// reuse operand allocation to the reused input. + fn process_operand_allocation( + &mut self, + inst: Inst, + op: Operand, + op_idx: usize, + ) -> Result<(), RegAllocError> { + if let Some(preg) = op.as_fixed_nonallocatable() { + self.allocs[(inst.index(), op_idx)] = Allocation::reg(preg); + trace!( + "Allocation for instruction {:?} and operand {}: {}", + inst, + op, + self.allocs[(inst.index(), op_idx)] + ); + return Ok(()); + } + if !self.allocd_within_constraint(op) { + trace!("{op} isn't allocated within constraints."); + let curr_alloc = self.vreg_allocs[op.vreg().vreg()]; + let new_alloc = self.alloc_operand(inst, op, op_idx)?; + if curr_alloc.is_none() { + self.live_vregs.insert(op.vreg()); + self.vreg_to_live_inst_range[op.vreg().vreg()].1 = match (op.pos(), op.kind()) { + (OperandPos::Late, OperandKind::Use) | (_, OperandKind::Def) => { + // Live range ends just before the early phase of the + // next instruction. + ProgPoint::before(Inst::new(inst.index() + 1)) + } + (OperandPos::Early, OperandKind::Use) => { + // Live range ends just before the late phase of the current instruction. + ProgPoint::after(inst) + } + }; + self.vreg_to_live_inst_range[op.vreg().vreg()].2 = new_alloc; + + trace!("Setting vreg_allocs[{op}] to {new_alloc:?}"); + self.vreg_allocs[op.vreg().vreg()] = new_alloc; + if let Some(preg) = new_alloc.as_reg() { + self.vreg_in_preg[preg.index()] = op.vreg(); + } + } + // Need to insert a move to propagate flow from the current + // allocation to the subsequent places where the value was + // used (in `prev_alloc`, that is). + else { + trace!("Move reason: Prev allocation doesn't meet constraints"); + if self.edits.is_stack(new_alloc) + && self.edits.is_stack(curr_alloc) + && self.edits.scratch_regs[op.class()].is_none() + { + self.alloc_scratch_reg(inst, op.class())?; + } + if op.kind() == OperandKind::Def { + trace!("Adding edit from {new_alloc:?} to {curr_alloc:?} after inst {inst:?} for {op}"); + self.edits.add_move( + inst, + new_alloc, + curr_alloc, + op.class(), + InstPosition::After, + ); + // No need to set vreg_in_preg because it will be set during + // `freealloc` if needed. + } + // Edits for use operands are added later to avoid inserting + // edits out of order. + + if let Some(preg) = new_alloc.as_reg() { + // Don't change the allocation. + self.vreg_in_preg[preg.index()] = VReg::invalid(); + } + } + trace!( + "Allocation for instruction {:?} and operand {}: {}", + inst, + op, + self.allocs[(inst.index(), op_idx)] + ); + } else { + trace!("{op} is already allocated within constraints"); + self.allocs[(inst.index(), op_idx)] = self.vreg_allocs[op.vreg().vreg()]; + if let Some(preg) = self.allocs[(inst.index(), op_idx)].as_reg() { + if self.allocatable_regs.contains(preg) { + self.lrus[preg.class()].poke(preg); + } + self.available_pregs[op.pos()].remove(preg); + match (op.pos(), op.kind()) { + (OperandPos::Late, OperandKind::Use) => { + self.available_pregs[OperandPos::Early].remove(preg); + } + (OperandPos::Early, OperandKind::Def) => { + self.available_pregs[OperandPos::Late].remove(preg); + } + _ => (), + }; + } + trace!( + "Allocation for instruction {:?} and operand {}: {}", + inst, + op, + self.allocs[(inst.index(), op_idx)] + ); + } + trace!( + "Late available regs: {}", + self.available_pregs[OperandPos::Late] + ); + trace!( + "Early available regs: {}", + self.available_pregs[OperandPos::Early] + ); + Ok(()) + } + + fn remove_clobbers_from_available_pregs(&mut self, clobbers: PRegSet) { + trace!("Removing clobbers {clobbers} from available reg sets"); + // Don't let defs get allocated to clobbers. + // Consider a scenario: + // + // 1. (early|late) def v0 (reg). Clobbers: [p0] + // 2. use v0 (fixed: p0) + // + // If p0 isn't removed from the both available reg sets, then + // p0 could get allocated to v0 in inst 1, making it impossible + // to restore it after the instruction. + // To avoid this scenario, clobbers should be removed from both late + // and early reg sets. + let all_but_clobbers = clobbers.invert(); + self.available_pregs[OperandPos::Late].intersect_from(all_but_clobbers); + self.available_pregs[OperandPos::Early].intersect_from(all_but_clobbers); + } + + /// If instruction `inst` is a branch in `block`, + /// this function places branch arguments in the spillslots + /// expected by the destination blocks. + fn process_branch(&mut self, block: Block, inst: Inst) -> Result<(), RegAllocError> { + use OperandPos::*; + trace!("Processing branch instruction {inst:?} in block {block:?}"); + + let mut int_parallel_moves = ParallelMoves::new(); + let mut float_parallel_moves = ParallelMoves::new(); + let mut vec_parallel_moves = ParallelMoves::new(); + + for (succ_idx, succ) in self.func.block_succs(block).iter().enumerate() { + for (pos, vreg) in self + .func + .branch_blockparams(block, inst, succ_idx) + .iter() + .enumerate() + { + let succ_params = self.func.block_params(*succ); + let succ_param_vreg = succ_params[pos]; + if self.vreg_spillslots[succ_param_vreg.vreg()].is_invalid() { + self.vreg_spillslots[succ_param_vreg.vreg()] = + self.stack.allocstack(succ_param_vreg.class()); + } + if self.vreg_spillslots[vreg.vreg()].is_invalid() { + self.vreg_spillslots[vreg.vreg()] = self.stack.allocstack(vreg.class()); + } + let vreg_spill = Allocation::stack(self.vreg_spillslots[vreg.vreg()]); + let curr_alloc = self.vreg_allocs[vreg.vreg()]; + if curr_alloc.is_none() { + self.live_vregs.insert(*vreg); + self.vreg_to_live_inst_range[vreg.vreg()].1 = ProgPoint::before(inst); + } else if curr_alloc != vreg_spill { + if self.edits.is_stack(curr_alloc) + && self.edits.scratch_regs[vreg.class()].is_none() + { + let reg = self.get_scratch_reg_for_reload( + inst, + vreg.class(), + self.available_pregs[Early] & self.available_pregs[Late], + )?; + self.edits.scratch_regs[vreg.class()] = Some(reg); + self.available_pregs[OperandPos::Early].remove(reg); + self.available_pregs[OperandPos::Late].remove(reg); + } + self.edits.add_move( + inst, + vreg_spill, + curr_alloc, + vreg.class(), + InstPosition::Before, + ); + } + self.vreg_allocs[vreg.vreg()] = vreg_spill; + let parallel_moves = match vreg.class() { + RegClass::Int => &mut int_parallel_moves, + RegClass::Float => &mut float_parallel_moves, + RegClass::Vector => &mut vec_parallel_moves, + }; + let from = Allocation::stack(self.vreg_spillslots[vreg.vreg()]); + let to = Allocation::stack(self.vreg_spillslots[succ_param_vreg.vreg()]); + trace!("Recording parallel move from {from} to {to}"); + parallel_moves.add(from, to, Some(*vreg)); + } + } + + let resolved_int = int_parallel_moves.resolve(); + let resolved_float = float_parallel_moves.resolve(); + let resolved_vec = vec_parallel_moves.resolve(); + let mut scratch_regs = self.edits.scratch_regs.clone(); + let mut num_spillslots = self.stack.num_spillslots; + let mut avail_regs = self.available_pregs[Early] & self.available_pregs[Late]; + + trace!("Resolving parallel moves"); + for (resolved, class) in [ + (resolved_int, RegClass::Int), + (resolved_float, RegClass::Float), + (resolved_vec, RegClass::Vector), + ] { + let scratch_resolver = MoveAndScratchResolver { + find_free_reg: || { + if let Some(reg) = scratch_regs[class] { + trace!("Retrieved reg {reg} for scratch resolver"); + scratch_regs[class] = None; + Some(Allocation::reg(reg)) + } else { + let Some(preg) = self.lrus[class].last(avail_regs) else { + trace!("Couldn't find any reg for scratch resolver"); + return None; + }; + avail_regs.remove(preg); + trace!("Retrieved reg {preg} for scratch resolver"); + Some(Allocation::reg(preg)) + } + }, + get_stackslot: || { + let size: u32 = self.func.spillslot_size(class).try_into().unwrap(); + let mut offset = num_spillslots; + debug_assert!(size.is_power_of_two()); + offset = (offset + size - 1) & !(size - 1); + let slot = if self.func.multi_spillslot_named_by_last_slot() { + offset + size - 1 + } else { + offset + }; + offset += size; + num_spillslots = offset; + trace!("Retrieved slot {slot} for scratch resolver"); + Allocation::stack(SpillSlot::new(slot as usize)) + }, + is_stack_alloc: |alloc| self.edits.is_stack(alloc), + borrowed_scratch_reg: self.preferred_victim[class], + }; + let moves = scratch_resolver.compute(resolved); + trace!("Resolved {class:?} parallel moves"); + for (from, to, _) in moves.into_iter().rev() { + self.edits + .edits + .push((ProgPoint::before(inst), Edit::Move { from, to })) + } + self.stack.num_spillslots = num_spillslots; + } + trace!("Completed processing branch"); + Ok(()) + } + + fn alloc_inst(&mut self, block: Block, inst: Inst) -> Result<(), RegAllocError> { + trace!("Allocating instruction {:?}", inst); + self.reset_available_pregs_and_scratch_regs(); + let operands = Operands::new(self.func.inst_operands(inst)); + let clobbers = self.func.inst_clobbers(inst); + + for (op_idx, op) in operands.reuse() { + trace!("Initializing reused_input_to_reuse_op for {op}"); + let OperandConstraint::Reuse(reused_idx) = op.constraint() else { + unreachable!() + }; + self.reused_input_to_reuse_op[reused_idx] = op_idx; + } + for (op_idx, op) in operands.fixed() { + let OperandConstraint::FixedReg(preg) = op.constraint() else { + unreachable!(); + }; + self.reserve_reg_for_fixed_operand(op, op_idx, preg)?; + if self.allocatable_regs.contains(preg) { + self.lrus[preg.class()].poke(preg); + } + } + for (_, op) in operands.fixed() { + let OperandConstraint::FixedReg(preg) = op.constraint() else { + unreachable!(); + }; + // Eviction has to be done separately to avoid using a fixed register + // as a scratch register. + if self.vreg_in_preg[preg.index()] != VReg::invalid() + && self.vreg_in_preg[preg.index()] != op.vreg() + { + trace!( + "Evicting {} from fixed register {preg}", + self.vreg_in_preg[preg.index()] + ); + if self.fixed_stack_slots.contains(preg) + && self.edits.scratch_regs[preg.class()].is_none() + { + self.alloc_scratch_reg(inst, preg.class())?; + } + self.evict_vreg_in_preg(inst, preg); + self.vreg_in_preg[preg.index()] = VReg::invalid(); + } + } + self.remove_clobbers_from_available_pregs(clobbers); + for preg in clobbers { + if self.vreg_in_preg[preg.index()] != VReg::invalid() { + trace!( + "Evicting {} from clobber {preg}", + self.vreg_in_preg[preg.index()] + ); + if self.fixed_stack_slots.contains(preg) + && self.edits.scratch_regs[preg.class()].is_none() + { + self.alloc_scratch_reg(inst, preg.class())?; + } + self.evict_vreg_in_preg(inst, preg); + self.vreg_in_preg[preg.index()] = VReg::invalid(); + } + } + for (op_idx, op) in operands.def_ops() { + trace!("Allocating def operands {op}"); + if let OperandConstraint::Reuse(reused_idx) = op.constraint() { + let reused_op = operands[reused_idx]; + let new_reuse_op = + Operand::new(op.vreg(), reused_op.constraint(), op.kind(), op.pos()); + trace!("allocating reuse op {op} as {new_reuse_op}"); + self.process_operand_allocation(inst, new_reuse_op, op_idx)?; + } else { + self.process_operand_allocation(inst, op, op_idx)?; + } + let slot = self.vreg_spillslots[op.vreg().vreg()]; + if slot.is_valid() { + self.vreg_to_live_inst_range[op.vreg().vreg()].2 = Allocation::stack(slot); + let curr_alloc = self.vreg_allocs[op.vreg().vreg()]; + let vreg_slot = self.vreg_spillslots[op.vreg().vreg()]; + let (is_stack_to_stack, src_and_dest_are_same) = + if let Some(curr_alloc) = curr_alloc.as_stack() { + (true, curr_alloc == vreg_slot) + } else { + (self.edits.is_stack(curr_alloc), false) + }; + if !src_and_dest_are_same { + if is_stack_to_stack && self.edits.scratch_regs[op.class()].is_none() { + self.alloc_scratch_reg(inst, op.class())?; + }; + self.edits.add_move( + inst, + self.vreg_allocs[op.vreg().vreg()], + Allocation::stack(self.vreg_spillslots[op.vreg().vreg()]), + op.class(), + InstPosition::After, + ); + } + } + self.vreg_to_live_inst_range[op.vreg().vreg()].0 = ProgPoint::after(inst); + self.freealloc(op.vreg()); + } + for (op_idx, op) in operands.use_ops() { + trace!("Allocating use operand {op}"); + if self.reused_input_to_reuse_op[op_idx] != usize::MAX { + let reuse_op_idx = self.reused_input_to_reuse_op[op_idx]; + let reuse_op_alloc = self.allocs[(inst.index(), reuse_op_idx)]; + let Some(preg) = reuse_op_alloc.as_reg() else { + unreachable!(); + }; + let new_reused_input_constraint = OperandConstraint::FixedReg(preg); + let new_reused_input = + Operand::new(op.vreg(), new_reused_input_constraint, op.kind(), op.pos()); + trace!("Allocating reused input {op} as {new_reused_input}"); + self.process_operand_allocation(inst, new_reused_input, op_idx)?; + } else { + self.process_operand_allocation(inst, op, op_idx)?; + } + } + for (op_idx, op) in operands.use_ops() { + if op.as_fixed_nonallocatable().is_some() { + continue; + } + if self.vreg_allocs[op.vreg().vreg()] != self.allocs[(inst.index(), op_idx)] { + let curr_alloc = self.vreg_allocs[op.vreg().vreg()]; + let new_alloc = self.allocs[(inst.index(), op_idx)]; + trace!("Adding edit from {curr_alloc:?} to {new_alloc:?} before inst {inst:?} for {op}"); + self.edits.add_move( + inst, + curr_alloc, + new_alloc, + op.class(), + InstPosition::Before, + ); + } + } + if self.func.is_branch(inst) { + self.process_branch(block, inst)?; + } + for entry in self.reused_input_to_reuse_op.iter_mut() { + *entry = usize::MAX; + } + if trace_enabled!() { + self.log_post_inst_processing_state(inst); + } + Ok(()) + } + + /// At the beginning of every block, all virtual registers that are + /// livein are expected to be in their respective spillslots. + /// This function sets the current allocations of livein registers + /// to their spillslots and inserts the edits to flow livein values to + /// the allocations where they are expected to be before the first + /// instruction. + fn reload_at_begin(&mut self, block: Block) -> Result<(), RegAllocError> { + trace!( + "Reloading live registers at the beginning of block {:?}", + block + ); + trace!( + "Live registers at the beginning of block {:?}: {:?}", + block, + self.live_vregs + ); + trace!( + "Block params at block {:?} beginning: {:?}", + block, + self.func.block_params(block) + ); + trace!( + "Available pregs: {}", + self.available_pregs[OperandPos::Early] + ); + self.reset_available_pregs_and_scratch_regs(); + let avail_regs_for_scratch = self.available_pregs[OperandPos::Early]; + let first_inst = self.func.block_insns(block).first(); + // We need to check for the registers that are still live. + // These registers are either livein or block params + // Liveins should be stack-allocated and block params should be freed. + for vreg in self.func.block_params(block).iter().cloned() { + trace!("Processing {}", vreg); + if self.vreg_allocs[vreg.vreg()] == Allocation::none() { + // If this block param was never used, its allocation will + // be none at this point. + continue; + } + if self.vreg_spillslots[vreg.vreg()].is_invalid() { + self.vreg_spillslots[vreg.vreg()] = self.stack.allocstack(vreg.class()); + } + // The allocation where the vreg is expected to be before + // the first instruction. + let prev_alloc = self.vreg_allocs[vreg.vreg()]; + let slot = Allocation::stack(self.vreg_spillslots[vreg.vreg()]); + self.vreg_to_live_inst_range[vreg.vreg()].2 = slot; + self.vreg_to_live_inst_range[vreg.vreg()].0 = ProgPoint::before(first_inst); + trace!("{} is a block param. Freeing it", vreg); + // A block's block param is not live before the block. + // And `vreg_allocs[i]` of a virtual register i is none for + // dead vregs. + self.freealloc(vreg); + if slot == prev_alloc { + // No need to do any movements if the spillslot is where the vreg is expected to be. + trace!( + "No need to reload {} because it's already in its expected allocation", + vreg + ); + continue; + } + trace!( + "Move reason: reload {} at begin - move from its spillslot", + vreg + ); + if self.edits.is_stack(prev_alloc) && self.edits.scratch_regs[vreg.class()].is_none() { + let reg = self.get_scratch_reg_for_reload( + first_inst, + vreg.class(), + avail_regs_for_scratch, + )?; + self.edits.scratch_regs[vreg.class()] = Some(reg); + } + self.edits.add_move( + self.func.block_insns(block).first(), + slot, + prev_alloc, + vreg.class(), + InstPosition::Before, + ); + } + for vreg in self.live_vregs.iter() { + trace!("Processing {}", vreg); + trace!( + "{} is not a block param. It's a liveout vreg from some predecessor", + vreg + ); + if self.vreg_spillslots[vreg.vreg()].is_invalid() { + self.vreg_spillslots[vreg.vreg()] = self.stack.allocstack(vreg.class()); + } + // The allocation where the vreg is expected to be before + // the first instruction. + let prev_alloc = self.vreg_allocs[vreg.vreg()]; + let slot = Allocation::stack(self.vreg_spillslots[vreg.vreg()]); + trace!("Setting {}'s current allocation to its spillslot", vreg); + self.vreg_allocs[vreg.vreg()] = slot; + if let Some(preg) = prev_alloc.as_reg() { + trace!("{} was in {}. Removing it", preg, vreg); + // Nothing is in that preg anymore. + self.vreg_in_preg[preg.index()] = VReg::invalid(); + } + if slot == prev_alloc { + // No need to do any movements if the spillslot is where the vreg is expected to be. + trace!( + "No need to reload {} because it's already in its expected allocation", + vreg + ); + continue; + } + trace!( + "Move reason: reload {} at begin - move from its spillslot", + vreg + ); + if self.edits.is_stack(prev_alloc) && self.edits.scratch_regs[vreg.class()].is_none() { + let Some(preg) = self.lrus[vreg.class()].last(avail_regs_for_scratch) else { + return Err(RegAllocError::TooManyLiveRegs); + }; + if self.vreg_in_preg[preg.index()] != VReg::invalid() { + // Had to put `evict_reg_in_preg_before_inst` here because of borrow checker rules. + trace!("Removing the vreg in preg {} for eviction", preg); + let evicted_vreg = self.vreg_in_preg[preg.index()]; + trace!("The removed vreg: {}", evicted_vreg); + debug_assert_ne!(evicted_vreg, VReg::invalid()); + if self.vreg_spillslots[evicted_vreg.vreg()].is_invalid() { + self.vreg_spillslots[evicted_vreg.vreg()] = + self.stack.allocstack(evicted_vreg.class()); + } + let slot = self.vreg_spillslots[evicted_vreg.vreg()]; + self.vreg_allocs[evicted_vreg.vreg()] = Allocation::stack(slot); + trace!("Move reason: eviction"); + self.edits.add_move( + first_inst, + self.vreg_allocs[evicted_vreg.vreg()], + Allocation::reg(preg), + evicted_vreg.class(), + InstPosition::Before, + ); + } + self.edits.scratch_regs[vreg.class()] = Some(preg); + } + self.edits.add_move( + first_inst, + slot, + prev_alloc, + vreg.class(), + InstPosition::Before, + ); + } + if trace_enabled!() { + self.log_post_reload_at_begin_state(block); + } + Ok(()) + } + + fn log_post_reload_at_begin_state(&self, block: Block) { + use alloc::format; + trace!(""); + trace!("State after instruction reload_at_begin of {:?}", block); + let mut map = FxHashMap::default(); + for (vreg_idx, alloc) in self.vreg_allocs.iter().enumerate() { + if *alloc != Allocation::none() { + map.insert(format!("vreg{vreg_idx}"), alloc); + } + } + trace!("vreg_allocs: {:?}", map); + let mut map = FxHashMap::default(); + for i in 0..self.vreg_in_preg.len() { + if self.vreg_in_preg[i] != VReg::invalid() { + map.insert(PReg::from_index(i), self.vreg_in_preg[i]); + } + } + trace!("vreg_in_preg: {:?}", map); + trace!("Int LRU: {:?}", self.lrus[RegClass::Int]); + trace!("Float LRU: {:?}", self.lrus[RegClass::Float]); + trace!("Vector LRU: {:?}", self.lrus[RegClass::Vector]); + } + + fn log_post_inst_processing_state(&self, inst: Inst) { + use alloc::format; + trace!(""); + trace!("State after instruction {:?}", inst); + let mut map = FxHashMap::default(); + for (vreg_idx, alloc) in self.vreg_allocs.iter().enumerate() { + if *alloc != Allocation::none() { + map.insert(format!("vreg{vreg_idx}"), alloc); + } + } + trace!("vreg_allocs: {:?}", map); + let mut v = Vec::new(); + for i in 0..self.vreg_in_preg.len() { + if self.vreg_in_preg[i] != VReg::invalid() { + v.push(format!( + "{}: {}, ", + PReg::from_index(i), + self.vreg_in_preg[i] + )); + } + } + trace!("vreg_in_preg: {:?}", v); + trace!("Int LRU: {:?}", self.lrus[RegClass::Int]); + trace!("Float LRU: {:?}", self.lrus[RegClass::Float]); + trace!("Vector LRU: {:?}", self.lrus[RegClass::Vector]); + trace!(""); + } + + fn alloc_block(&mut self, block: Block) -> Result<(), RegAllocError> { + trace!("{:?} start", block); + for inst in self.func.block_insns(block).iter().rev() { + self.alloc_inst(block, inst)?; + } + self.reload_at_begin(block)?; + trace!("{:?} end\n", block); + Ok(()) + } + + fn build_debug_info(&mut self) { + trace!("Building debug location info"); + for &(vreg, start, end, label) in self.func.debug_value_labels() { + let (point_start, point_end, alloc) = self.vreg_to_live_inst_range[vreg.vreg()]; + if point_start.inst() <= start && end <= point_end.inst().next() { + self.debug_locations + .push((label, point_start, point_end, alloc)); + } + } + self.debug_locations.sort_by_key(|loc| loc.0); + } + + fn run(&mut self) -> Result<(), RegAllocError> { + debug_assert_eq!(self.func.entry_block().index(), 0); + for block in (0..self.func.num_blocks()).rev() { + self.alloc_block(Block::new(block))?; + } + self.edits.edits.reverse(); + self.build_debug_info(); + // Ought to check if there are livein registers + // then throw an error, but will that be expensive? + Ok(()) + } +} + +fn log_function(func: &F) { + trace!("Processing a new function"); + for block in 0..func.num_blocks() { + let block = Block::new(block); + trace!( + "Block {:?}. preds: {:?}. succs: {:?}, params: {:?}", + block, + func.block_preds(block), + func.block_succs(block), + func.block_params(block) + ); + for inst in func.block_insns(block).iter() { + let clobbers = func.inst_clobbers(inst); + trace!( + "inst{:?}: {:?}. Clobbers: {}", + inst.index(), + func.inst_operands(inst), + clobbers + ); + if func.is_branch(inst) { + trace!("Block args: "); + for (succ_idx, _succ) in func.block_succs(block).iter().enumerate() { + trace!(" {:?}", func.branch_blockparams(block, inst, succ_idx)); + } + } + } + trace!(""); + } +} + +fn log_output<'a, F: Function>(env: &Env<'a, F>) { + trace!("Done!"); + use alloc::format; + let mut v = Vec::new(); + for i in 0..env.func.num_vregs() { + if env.vreg_spillslots[i].is_valid() { + v.push(( + format!("{}", VReg::new(i, RegClass::Int)), + format!("{}", Allocation::stack(env.vreg_spillslots[i])), + )); + } + } + trace!("VReg spillslots: {:?}", v); + trace!("Final edits: {:?}", env.edits.edits); +} + +pub fn run( + func: &F, + mach_env: &MachineEnv, + verbose_log: bool, + enable_ssa_checker: bool, +) -> Result { + if enable_ssa_checker { + let mut cfginfo = CFGInfo::default(); + cfginfo.init(func)?; + validate_ssa(func, &cfginfo)?; + } + + if trace_enabled!() || verbose_log { + log_function(func); + } + + let mut env = Env::new(func, mach_env); + env.run()?; + + if trace_enabled!() || verbose_log { + log_output(&env); + } + + Ok(Output { + edits: env.edits.edits, + allocs: env.allocs.allocs, + inst_alloc_offsets: env.allocs.inst_alloc_offsets, + num_spillslots: env.stack.num_spillslots as usize, + debug_locations: env.debug_locations, + stats: env.stats, + }) +} diff --git a/src/fastalloc/tests.rs b/src/fastalloc/tests.rs new file mode 100644 index 00000000..41be3771 --- /dev/null +++ b/src/fastalloc/tests.rs @@ -0,0 +1,315 @@ +use crate::OperandConstraint::{self, *}; +use crate::OperandKind::{self, *}; +use crate::{ + run, Algorithm, Allocation, Block, Function, Inst, InstRange, MachineEnv, Operand, OperandPos, + PReg, PRegSet, ProgPoint, RegClass, RegallocOptions, VReg, +}; +use alloc::vec; +use alloc::vec::Vec; + +#[test] +fn test_debug_locations1() { + let mach_env = mach_env(10); + let mut options = RegallocOptions::default(); + options.validate_ssa = true; + options.algorithm = Algorithm::Fastalloc; + let mut f = RealFunction::new(vec![BlockBuildInfo { + insts: vec![ + /* 0. */ vec![op(Def, 0, FixedReg(p(0)))], + /* 1. */ + vec![ + op(Def, 1, FixedReg(p(0))), + op(Use, 0, FixedReg(p(0))), + op(Use, 0, Reg), + ], + /* 2. */ + vec![ + op(Def, 2, FixedReg(p(8))), + op(Use, 0, FixedReg(p(2))), + op(Use, 1, FixedReg(p(0))), + ], + /* 3. */ vec![op(Def, 3, FixedReg(p(9))), op(Use, 0, FixedReg(p(9)))], + ], + }]); + f.debug_value_labels = vec![ + (v(0), i(0), i(4), 32), + (v(2), i(2), i(4), 70), + (v(2), i(2), i(4), 71), + (v(3), i(3), i(4), 34), + ]; + let result = run(&f, &mach_env, &options).unwrap(); + assert_eq!( + result.debug_locations, + vec![ + ( + 32, + ProgPoint::after(i(0)), + ProgPoint::after(i(3)), + alloc(p(9)) + ), + ( + 34, + ProgPoint::after(i(3)), + ProgPoint::before(i(4)), + alloc(p(9)) + ), + ( + 70, + ProgPoint::after(i(2)), + ProgPoint::before(i(3)), + alloc(p(8)) + ), + ( + 71, + ProgPoint::after(i(2)), + ProgPoint::before(i(3)), + alloc(p(8)) + ), + ] + ); +} + +#[test] +fn test_debug_locations2() { + let mach_env = mach_env(2); + let mut options = RegallocOptions::default(); + options.validate_ssa = true; + options.algorithm = Algorithm::Fastalloc; + let mut f = RealFunction::new(vec![BlockBuildInfo { + insts: vec![ + /* 0. */ vec![op(Def, 2, FixedReg(p(0)))], + /* 1. */ vec![op(Def, 0, FixedReg(p(0)))], + /* 2. */ vec![op(Def, 1, FixedReg(p(1)))], + /* 3. */ vec![op(Use, 0, FixedReg(p(0))), op(Use, 0, FixedReg(p(1)))], + /* 4. */ vec![op(Use, 1, FixedReg(p(1)))], + ], + }]); + f.debug_value_labels = vec![ + (v(0), i(1), i(4), 10), + (v(1), i(0), i(1), 11), + (v(1), i(2), i(3), 23), + ]; + let result = run(&f, &mach_env, &options).unwrap(); + assert_eq!(result.debug_locations.len(), 2); + assert_eq!( + result.debug_locations[0], + ( + 10, + ProgPoint::after(i(1)), + ProgPoint::after(i(3)), + alloc(p(0)) + ) + ); + assert_eq!(result.debug_locations[1].0, 23); + assert_eq!(result.debug_locations[1].1, ProgPoint::after(i(2))); + assert_eq!(result.debug_locations[1].2, ProgPoint::after(i(4))); + assert!(matches!(result.debug_locations[1].3.as_stack(), Some(_))); +} + +impl RealFunction { + fn new(blocks: Vec) -> Self { + assert!(blocks.len() <= 2, "Just for testing purposes"); + let mut f = Self::default(); + let mut max_vreg_num_seen = 0; + for block in blocks.iter() { + f.blocks.push(RealBlock { + params: vec![], + preds: vec![], + succs: vec![], + }); + let start_inst_idx = f.insts.len(); + for inst in block.insts.iter() { + f.insts.push(RealInst { + inst: Inst::new(f.insts.len()), + kind: RealInstKind::Normal, + }); + let start_op_idx = f.operands.len(); + for op in inst.iter() { + max_vreg_num_seen = max_vreg_num_seen.max(op.vreg().vreg()); + f.operands.push(*op); + } + f.operand_ranges.push((start_op_idx, f.operands.len())); + } + if !block.insts.is_empty() { + f.insts.last_mut().unwrap().kind = RealInstKind::Ret; + } + f.inst_ranges.push((start_inst_idx, f.insts.len())); + } + f.num_vregs = max_vreg_num_seen + 1; + f + } +} + +fn mach_env(no_of_regs: usize) -> MachineEnv { + MachineEnv { + preferred_regs_by_class: [ + (0..no_of_regs) + .map(|no| PReg::new(no, RegClass::Int)) + .collect(), + vec![], + vec![], + ], + non_preferred_regs_by_class: [vec![], vec![], vec![]], + scratch_by_class: [None, None, None], + fixed_stack_slots: vec![], + } +} + +fn op(kind: OperandKind, vreg_num: usize, constraint: OperandConstraint) -> Operand { + Operand::new( + VReg::new(vreg_num, RegClass::Int), + constraint, + kind, + match kind { + Use => OperandPos::Early, + Def => OperandPos::Late, + }, + ) +} + +fn alloc(preg: PReg) -> Allocation { + Allocation::reg(preg) +} + +fn v(vreg_num: usize) -> VReg { + VReg::new(vreg_num, RegClass::Int) +} + +fn i(inst: usize) -> Inst { + Inst::new(inst) +} + +fn p(hw_enc: usize) -> PReg { + PReg::new(hw_enc, RegClass::Int) +} + +struct BlockBuildInfo { + insts: Vec>, +} + +#[derive(Default)] +struct RealFunction { + blocks: Vec, + insts: Vec, + operands: Vec, + operand_ranges: Vec<(usize, usize)>, + inst_ranges: Vec<(usize, usize)>, + num_vregs: usize, + debug_value_labels: Vec<(VReg, Inst, Inst, u32)>, +} + +struct RealBlock { + params: Vec, + preds: Vec, + succs: Vec, +} + +struct RealInst { + inst: Inst, + kind: RealInstKind, +} + +impl RealInst { + fn is_branch(&self) -> bool { + match self.kind { + RealInstKind::Branch(_, _) => true, + _ => false, + } + } + + fn is_ret(&self) -> bool { + match self.kind { + RealInstKind::Ret => true, + _ => false, + } + } +} + +enum RealInstKind { + Normal, + Branch(Block, Vec), + Ret, +} + +impl Function for RealFunction { + fn num_insts(&self) -> usize { + self.insts.len() + } + + fn num_blocks(&self) -> usize { + self.blocks.len() + } + + fn block_insns(&self, block: crate::Block) -> crate::InstRange { + let (start, end) = self.inst_ranges[block.index()]; + if start != end { + InstRange::new( + self.insts[start].inst, + Inst::new(self.insts[end - 1].inst.index() + 1), + ) + } else { + InstRange::new(Inst::new(0), Inst::new(0)) + } + } + + fn allow_multiple_vreg_defs(&self) -> bool { + false + } + + fn block_params(&self, block: crate::Block) -> &[VReg] { + &self.blocks[block.index()].params + } + + fn block_preds(&self, block: crate::Block) -> &[crate::Block] { + &self.blocks[block.index()].preds + } + + fn block_succs(&self, block: Block) -> &[Block] { + &self.blocks[block.index()].succs + } + + fn debug_value_labels(&self) -> &[(VReg, Inst, Inst, u32)] { + &self.debug_value_labels + } + + fn entry_block(&self) -> Block { + Block::new(0) + } + + fn inst_clobbers(&self, _insn: Inst) -> crate::PRegSet { + PRegSet::empty() + } + + fn inst_operands(&self, insn: Inst) -> &[Operand] { + let (start, end) = self.operand_ranges[insn.index()]; + &self.operands[start..end] + } + + fn is_branch(&self, insn: Inst) -> bool { + self.insts[insn.index()].is_branch() + } + + fn is_ret(&self, insn: Inst) -> bool { + self.insts[insn.index()].is_ret() + } + + fn multi_spillslot_named_by_last_slot(&self) -> bool { + false + } + + fn num_vregs(&self) -> usize { + self.num_vregs + } + + fn spillslot_size(&self, regclass: crate::RegClass) -> usize { + match regclass { + RegClass::Int => 2, + RegClass::Float => 4, + RegClass::Vector => 8, + } + } + + fn branch_blockparams(&self, _block: Block, _insn: Inst, _succ_idx: usize) -> &[VReg] { + &[] + } +} diff --git a/src/fastalloc/vregset.rs b/src/fastalloc/vregset.rs new file mode 100644 index 00000000..77287dcc --- /dev/null +++ b/src/fastalloc/vregset.rs @@ -0,0 +1,143 @@ +use core::fmt; + +use crate::ion::data_structures::VRegIndex; +use crate::VReg; +use alloc::vec; +use alloc::vec::Vec; + +#[derive(Clone)] +struct VRegNode { + next: VRegIndex, + prev: VRegIndex, + vreg: VReg, +} + +// Using a doubly linked list here for fast insertion, +// removal and iteration. +pub struct VRegSet { + items: Vec, + head: VRegIndex, +} + +impl VRegSet { + pub fn with_capacity(num_vregs: usize) -> Self { + Self { + items: vec![ + VRegNode { + prev: VRegIndex::new(num_vregs), + next: VRegIndex::new(num_vregs), + vreg: VReg::invalid() + }; + num_vregs + 1 + ], + head: VRegIndex::new(num_vregs), + } + } + + pub fn insert(&mut self, vreg: VReg) { + debug_assert_eq!(self.items[vreg.vreg()].vreg, VReg::invalid()); + let old_head_next = self.items[self.head.index()].next; + self.items[vreg.vreg()] = VRegNode { + next: old_head_next, + prev: self.head, + vreg, + }; + self.items[self.head.index()].next = VRegIndex::new(vreg.vreg()); + self.items[old_head_next.index()].prev = VRegIndex::new(vreg.vreg()); + } + + pub fn remove(&mut self, vreg_num: usize) { + let prev = self.items[vreg_num].prev; + let next = self.items[vreg_num].next; + self.items[prev.index()].next = next; + self.items[next.index()].prev = prev; + self.items[vreg_num].vreg = VReg::invalid(); + } + + pub fn is_empty(&self) -> bool { + self.items[self.head.index()].next == self.head + } + + pub fn iter(&self) -> VRegSetIter { + VRegSetIter { + curr_item: self.items[self.head.index()].next, + head: self.head, + items: &self.items, + } + } +} + +pub struct VRegSetIter<'a> { + curr_item: VRegIndex, + head: VRegIndex, + items: &'a [VRegNode], +} + +impl<'a> Iterator for VRegSetIter<'a> { + type Item = VReg; + + fn next(&mut self) -> Option { + if self.curr_item != self.head { + let item = self.items[self.curr_item.index()].clone(); + self.curr_item = item.next; + Some(item.vreg) + } else { + None + } + } +} + +impl fmt::Debug for VRegSet { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{{ ")?; + for vreg in self.iter() { + write!(f, "{vreg} ")?; + } + write!(f, "}}") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::RegClass; + use RegClass::*; + const VREG: fn(usize, RegClass) -> VReg = VReg::new; + + #[test] + fn operations() { + let mut set = VRegSet::with_capacity(3090); + assert!(set.is_empty()); + set.insert(VREG(10, Int)); + set.insert(VREG(2000, Int)); + set.insert(VREG(11, Vector)); + set.insert(VREG(199, Float)); + set.insert(VREG(23, Int)); + let mut iter = set.iter(); + assert_eq!(iter.next(), Some(VREG(23, Int))); + assert_eq!(iter.next(), Some(VREG(199, Float))); + assert_eq!(iter.next(), Some(VREG(11, Vector))); + assert_eq!(iter.next(), Some(VREG(2000, Int))); + assert_eq!(iter.next(), Some(VREG(10, Int))); + + set.remove(23); + set.remove(11); + set.insert(VREG(73, Vector)); + let mut iter = set.iter(); + assert_eq!(iter.next(), Some(VREG(73, Vector))); + assert_eq!(iter.next(), Some(VREG(199, Float))); + assert_eq!(iter.next(), Some(VREG(2000, Int))); + assert_eq!(iter.next(), Some(VREG(10, Int))); + assert!(!set.is_empty()); + } + + #[test] + fn empty() { + let mut set = VRegSet::with_capacity(2000); + assert!(set.is_empty()); + set.insert(VREG(100, Int)); + assert!(!set.is_empty()); + set.remove(100); + assert!(set.is_empty()); + } +} diff --git a/src/fuzzing/mod.rs b/src/fuzzing/mod.rs index 4e5573fd..1aa619ec 100644 --- a/src/fuzzing/mod.rs +++ b/src/fuzzing/mod.rs @@ -24,6 +24,9 @@ pub mod cfg { pub mod ion { pub use crate::ion::*; } +pub mod fastalloc { + pub use crate::fastalloc::*; +} pub mod checker { pub use crate::checker::*; } diff --git a/src/lib.rs b/src/lib.rs index fdb1b3ac..61d20c5f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -35,13 +35,14 @@ macro_rules! trace_enabled { }; } -use core::hash::BuildHasherDefault; +use core::{hash::BuildHasherDefault, iter::FromIterator}; use rustc_hash::FxHasher; type FxHashMap = hashbrown::HashMap>; type FxHashSet = hashbrown::HashSet>; pub(crate) mod cfg; pub(crate) mod domtree; +pub(crate) mod fastalloc; pub mod indexset; pub(crate) mod ion; pub(crate) mod moves; @@ -258,6 +259,44 @@ impl PRegSet { self.bits[i] |= other.bits[i]; } } + + pub fn intersect_from(&mut self, other: PRegSet) { + for i in 0..self.bits.len() { + self.bits[i] &= other.bits[i]; + } + } + + pub fn invert(&self) -> PRegSet { + let mut set = self.bits; + for i in 0..self.bits.len() { + set[i] = !self.bits[i]; + } + PRegSet { bits: set } + } + + pub fn is_empty(&self, regclass: RegClass) -> bool { + self.bits[regclass as usize] == 0 + } +} + +impl core::ops::BitAnd for PRegSet { + type Output = PRegSet; + + fn bitand(self, rhs: PRegSet) -> Self::Output { + let mut out = self; + out.intersect_from(rhs); + out + } +} + +impl core::ops::BitOr for PRegSet { + type Output = PRegSet; + + fn bitor(self, rhs: PRegSet) -> Self::Output { + let mut out = self; + out.union_from(rhs); + out + } } impl IntoIterator for PRegSet { @@ -312,6 +351,26 @@ impl From<&MachineEnv> for PRegSet { } } +impl FromIterator for PRegSet { + fn from_iter>(iter: T) -> Self { + let mut set = Self::default(); + for preg in iter { + set.add(preg); + } + set + } +} + +impl core::fmt::Display for PRegSet { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{{")?; + for preg in self.into_iter() { + write!(f, "{preg}, ")?; + } + write!(f, "}}") + } +} + /// A virtual register. Contains a virtual register number and a /// class. /// @@ -359,6 +418,17 @@ impl VReg { pub const fn invalid() -> Self { VReg::new(Self::MAX, RegClass::Int) } + + #[inline(always)] + pub const fn bits(self) -> usize { + self.bits as usize + } +} + +impl From for VReg { + fn from(value: u32) -> Self { + Self { bits: value } + } } impl core::fmt::Debug for VReg { @@ -1300,6 +1370,11 @@ impl ProgPoint { pub fn from_index(index: u32) -> Self { Self { bits: index } } + + #[inline(always)] + pub fn invalid() -> Self { + Self::before(Inst::new(usize::MAX)) + } } /// An instruction to insert into the program to perform some data movement. @@ -1516,9 +1591,16 @@ pub fn run( env: &MachineEnv, options: &RegallocOptions, ) -> Result { - let mut ctx = Ctx::default(); - run_with_ctx(func, env, options, &mut ctx)?; - Ok(ctx.output) + match options.algorithm { + Algorithm::Ion => { + let mut ctx = Ctx::default(); + run_with_ctx(func, env, options, &mut ctx)?; + Ok(ctx.output) + } + Algorithm::Fastalloc => { + fastalloc::run(func, env, options.verbose_log, options.validate_ssa) + } + } } /// Run the allocator. @@ -1531,6 +1613,13 @@ pub fn run_with_ctx( ion::run(func, env, ctx, options.verbose_log, options.validate_ssa) } +#[derive(Clone, Copy, Debug, Default)] +pub enum Algorithm { + #[default] + Ion, + Fastalloc, +} + /// Options for allocation. #[derive(Clone, Copy, Debug, Default)] pub struct RegallocOptions { @@ -1539,6 +1628,9 @@ pub struct RegallocOptions { /// Run the SSA validator before allocating registers. pub validate_ssa: bool, + + /// The register allocation algorithm to be used. + pub algorithm: Algorithm, } pub(crate) trait VecExt {