Skip to content

Commit

Permalink
DAG: Replace bitwidth with type in suffix in atomic tablegen ops (llv…
Browse files Browse the repository at this point in the history 
…m#94845)
  • Loading branch information
@arsenm @EthanLuisMcDonough
arsenm authored and EthanLuisMcDonough committed Aug 13, 2024
1 parent a39569b commit af517f9
Show file tree
Hide file tree
Showing 27 changed files with 649 additions and 650 deletions.
74 changes: 26 additions & 48 deletions llvm/include/llvm/Target/TargetSelectionDAG.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1680,60 +1680,38 @@ multiclass ternary_atomic_op_ord {
}
}

multiclass binary_atomic_op<SDNode atomic_op, bit IsInt = 1> {
def _8 : PatFrag<(ops node:$ptr, node:$val),
(atomic_op node:$ptr, node:$val)> {
let IsAtomic = true;
let MemoryVT = !if(IsInt, i8, ?);
}
def _16 : PatFrag<(ops node:$ptr, node:$val),
(atomic_op node:$ptr, node:$val)> {
let IsAtomic = true;
let MemoryVT = !if(IsInt, i16, f16);
}
def _32 : PatFrag<(ops node:$ptr, node:$val),
(atomic_op node:$ptr, node:$val)> {
let IsAtomic = true;
let MemoryVT = !if(IsInt, i32, f32);
}
def _64 : PatFrag<(ops node:$ptr, node:$val),
(atomic_op node:$ptr, node:$val)> {
let IsAtomic = true;
let MemoryVT = !if(IsInt, i64, f64);
multiclass binary_atomic_op<SDNode atomic_op> {
foreach vt = [ i8, i16, i32, i64 ] in {
def _#vt : PatFrag<(ops node:$ptr, node:$val),
(atomic_op node:$ptr, node:$val)> {
let IsAtomic = true;
let MemoryVT = vt;
}

defm NAME#_#vt : binary_atomic_op_ord;
}
}

defm NAME#_8 : binary_atomic_op_ord;
defm NAME#_16 : binary_atomic_op_ord;
defm NAME#_32 : binary_atomic_op_ord;
defm NAME#_64 : binary_atomic_op_ord;
multiclass binary_atomic_op_fp<SDNode atomic_op> {
foreach vt = [ f16, bf16, v2f16, v2bf16, f32, f64 ] in {
def _#vt : PatFrag<(ops node:$ptr, node:$val),
(atomic_op node:$ptr, node:$val)> {
let IsAtomic = true;
let MemoryVT = vt;
}
}
}

multiclass ternary_atomic_op<SDNode atomic_op> {
def _8 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(atomic_op node:$ptr, node:$cmp, node:$val)> {
let IsAtomic = true;
let MemoryVT = i8;
}
def _16 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(atomic_op node:$ptr, node:$cmp, node:$val)> {
let IsAtomic = true;
let MemoryVT = i16;
}
def _32 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(atomic_op node:$ptr, node:$cmp, node:$val)> {
let IsAtomic = true;
let MemoryVT = i32;
}
def _64 : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(atomic_op node:$ptr, node:$cmp, node:$val)> {
let IsAtomic = true;
let MemoryVT = i64;
foreach vt = [ i8, i16, i32, i64 ] in {
def _#vt : PatFrag<(ops node:$ptr, node:$cmp, node:$val),
(atomic_op node:$ptr, node:$cmp, node:$val)> {
let IsAtomic = true;
let MemoryVT = vt;
}

defm NAME#_#vt : ternary_atomic_op_ord;
}

defm NAME#_8 : ternary_atomic_op_ord;
defm NAME#_16 : ternary_atomic_op_ord;
defm NAME#_32 : ternary_atomic_op_ord;
defm NAME#_64 : ternary_atomic_op_ord;
}

defm atomic_load_add : binary_atomic_op<atomic_load_add>;
Expand Down
60 changes: 30 additions & 30 deletions llvm/lib/Target/AArch64/AArch64InstrFormats.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11887,79 +11887,79 @@ multiclass LDOPregister<bits<3> opc, string op, bits<1> Acq, bits<1> Rel,
// complex DAG for DstRHS.
let Predicates = [HasLSE] in
multiclass LDOPregister_patterns_ord_dag<string inst, string suffix, string op,
string size, dag SrcRHS, dag DstRHS> {
def : Pat<(!cast<PatFrag>(op#"_"#size#"_monotonic") GPR64sp:$Rn, SrcRHS),
ValueType vt, dag SrcRHS, dag DstRHS> {
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_monotonic") GPR64sp:$Rn, SrcRHS),
(!cast<Instruction>(inst # suffix) DstRHS, GPR64sp:$Rn)>;
def : Pat<(!cast<PatFrag>(op#"_"#size#"_acquire") GPR64sp:$Rn, SrcRHS),
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_acquire") GPR64sp:$Rn, SrcRHS),
(!cast<Instruction>(inst # "A" # suffix) DstRHS, GPR64sp:$Rn)>;
def : Pat<(!cast<PatFrag>(op#"_"#size#"_release") GPR64sp:$Rn, SrcRHS),
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_release") GPR64sp:$Rn, SrcRHS),
(!cast<Instruction>(inst # "L" # suffix) DstRHS, GPR64sp:$Rn)>;
def : Pat<(!cast<PatFrag>(op#"_"#size#"_acq_rel") GPR64sp:$Rn, SrcRHS),
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_acq_rel") GPR64sp:$Rn, SrcRHS),
(!cast<Instruction>(inst # "AL" # suffix) DstRHS, GPR64sp:$Rn)>;
def : Pat<(!cast<PatFrag>(op#"_"#size#"_seq_cst") GPR64sp:$Rn, SrcRHS),
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_seq_cst") GPR64sp:$Rn, SrcRHS),
(!cast<Instruction>(inst # "AL" # suffix) DstRHS, GPR64sp:$Rn)>;
}

multiclass LDOPregister_patterns_ord<string inst, string suffix, string op,
string size, dag RHS> {
defm : LDOPregister_patterns_ord_dag<inst, suffix, op, size, RHS, RHS>;
ValueType vt, dag RHS> {
defm : LDOPregister_patterns_ord_dag<inst, suffix, op, vt, RHS, RHS>;
}

multiclass LDOPregister_patterns_ord_mod<string inst, string suffix, string op,
string size, dag LHS, dag RHS> {
defm : LDOPregister_patterns_ord_dag<inst, suffix, op, size, LHS, RHS>;
ValueType vt, dag LHS, dag RHS> {
defm : LDOPregister_patterns_ord_dag<inst, suffix, op, vt, LHS, RHS>;
}

multiclass LDOPregister_patterns<string inst, string op> {
defm : LDOPregister_patterns_ord<inst, "X", op, "64", (i64 GPR64:$Rm)>;
defm : LDOPregister_patterns_ord<inst, "W", op, "32", (i32 GPR32:$Rm)>;
defm : LDOPregister_patterns_ord<inst, "H", op, "16", (i32 GPR32:$Rm)>;
defm : LDOPregister_patterns_ord<inst, "B", op, "8", (i32 GPR32:$Rm)>;
defm : LDOPregister_patterns_ord<inst, "X", op, i64, (i64 GPR64:$Rm)>;
defm : LDOPregister_patterns_ord<inst, "W", op, i32, (i32 GPR32:$Rm)>;
defm : LDOPregister_patterns_ord<inst, "H", op, i16, (i32 GPR32:$Rm)>;
defm : LDOPregister_patterns_ord<inst, "B", op, i8, (i32 GPR32:$Rm)>;
}

multiclass LDOPregister_patterns_mod<string inst, string op, string mod> {
defm : LDOPregister_patterns_ord_mod<inst, "X", op, "64",
defm : LDOPregister_patterns_ord_mod<inst, "X", op, i64,
(i64 GPR64:$Rm),
(i64 (!cast<Instruction>(mod#Xrr) XZR, GPR64:$Rm))>;
defm : LDOPregister_patterns_ord_mod<inst, "W", op, "32",
defm : LDOPregister_patterns_ord_mod<inst, "W", op, i32,
(i32 GPR32:$Rm),
(i32 (!cast<Instruction>(mod#Wrr) WZR, GPR32:$Rm))>;
defm : LDOPregister_patterns_ord_mod<inst, "H", op, "16",
defm : LDOPregister_patterns_ord_mod<inst, "H", op, i16,
(i32 GPR32:$Rm),
(i32 (!cast<Instruction>(mod#Wrr) WZR, GPR32:$Rm))>;
defm : LDOPregister_patterns_ord_mod<inst, "B", op, "8",
defm : LDOPregister_patterns_ord_mod<inst, "B", op, i8,
(i32 GPR32:$Rm),
(i32 (!cast<Instruction>(mod#Wrr) WZR, GPR32:$Rm))>;
}

let Predicates = [HasLSE] in
multiclass CASregister_patterns_ord_dag<string inst, string suffix, string op,
string size, dag OLD, dag NEW> {
def : Pat<(!cast<PatFrag>(op#"_"#size#"_monotonic") GPR64sp:$Rn, OLD, NEW),
ValueType vt, dag OLD, dag NEW> {
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_monotonic") GPR64sp:$Rn, OLD, NEW),
(!cast<Instruction>(inst # suffix) OLD, NEW, GPR64sp:$Rn)>;
def : Pat<(!cast<PatFrag>(op#"_"#size#"_acquire") GPR64sp:$Rn, OLD, NEW),
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_acquire") GPR64sp:$Rn, OLD, NEW),
(!cast<Instruction>(inst # "A" # suffix) OLD, NEW, GPR64sp:$Rn)>;
def : Pat<(!cast<PatFrag>(op#"_"#size#"_release") GPR64sp:$Rn, OLD, NEW),
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_release") GPR64sp:$Rn, OLD, NEW),
(!cast<Instruction>(inst # "L" # suffix) OLD, NEW, GPR64sp:$Rn)>;
def : Pat<(!cast<PatFrag>(op#"_"#size#"_acq_rel") GPR64sp:$Rn, OLD, NEW),
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_acq_rel") GPR64sp:$Rn, OLD, NEW),
(!cast<Instruction>(inst # "AL" # suffix) OLD, NEW, GPR64sp:$Rn)>;
def : Pat<(!cast<PatFrag>(op#"_"#size#"_seq_cst") GPR64sp:$Rn, OLD, NEW),
def : Pat<(!cast<PatFrag>(op#"_"#vt#"_seq_cst") GPR64sp:$Rn, OLD, NEW),
(!cast<Instruction>(inst # "AL" # suffix) OLD, NEW, GPR64sp:$Rn)>;
}

multiclass CASregister_patterns_ord<string inst, string suffix, string op,
string size, dag OLD, dag NEW> {
defm : CASregister_patterns_ord_dag<inst, suffix, op, size, OLD, NEW>;
ValueType vt, dag OLD, dag NEW> {
defm : CASregister_patterns_ord_dag<inst, suffix, op, vt, OLD, NEW>;
}

multiclass CASregister_patterns<string inst, string op> {
defm : CASregister_patterns_ord<inst, "X", op, "64",
defm : CASregister_patterns_ord<inst, "X", op, i64,
(i64 GPR64:$Rold), (i64 GPR64:$Rnew)>;
defm : CASregister_patterns_ord<inst, "W", op, "32",
defm : CASregister_patterns_ord<inst, "W", op, i32,
(i32 GPR32:$Rold), (i32 GPR32:$Rnew)>;
defm : CASregister_patterns_ord<inst, "H", op, "16",
defm : CASregister_patterns_ord<inst, "H", op, i16,
(i32 GPR32:$Rold), (i32 GPR32:$Rnew)>;
defm : CASregister_patterns_ord<inst, "B", op, "8",
defm : CASregister_patterns_ord<inst, "B", op, i8,
(i32 GPR32:$Rold), (i32 GPR32:$Rnew)>;
}

Expand Down
8 changes: 4 additions & 4 deletions llvm/lib/Target/AArch64/AArch64InstrGISel.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -346,16 +346,16 @@ let Predicates = [HasNEON] in {
}

let Predicates = [HasNoLSE] in {
def : Pat<(atomic_cmp_swap_8 GPR64:$addr, GPR32:$desired, GPR32:$new),
def : Pat<(atomic_cmp_swap_i8 GPR64:$addr, GPR32:$desired, GPR32:$new),
(CMP_SWAP_8 GPR64:$addr, GPR32:$desired, GPR32:$new)>;

def : Pat<(atomic_cmp_swap_16 GPR64:$addr, GPR32:$desired, GPR32:$new),
def : Pat<(atomic_cmp_swap_i16 GPR64:$addr, GPR32:$desired, GPR32:$new),
(CMP_SWAP_16 GPR64:$addr, GPR32:$desired, GPR32:$new)>;

def : Pat<(atomic_cmp_swap_32 GPR64:$addr, GPR32:$desired, GPR32:$new),
def : Pat<(atomic_cmp_swap_i32 GPR64:$addr, GPR32:$desired, GPR32:$new),
(CMP_SWAP_32 GPR64:$addr, GPR32:$desired, GPR32:$new)>;

def : Pat<(atomic_cmp_swap_64 GPR64:$addr, GPR64:$desired, GPR64:$new),
def : Pat<(atomic_cmp_swap_i64 GPR64:$addr, GPR64:$desired, GPR64:$new),
(CMP_SWAP_64 GPR64:$addr, GPR64:$desired, GPR64:$new)>;
}

Expand Down
31 changes: 22 additions & 9 deletions llvm/lib/Target/AMDGPU/AMDGPUInstructions.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -637,21 +637,36 @@ defm int_amdgcn_atomic_cond_sub_u32 : local_addr_space_atomic_op;
defm int_amdgcn_atomic_cond_sub_u32 : flat_addr_space_atomic_op;
defm int_amdgcn_atomic_cond_sub_u32 : global_addr_space_atomic_op;

multiclass noret_binary_atomic_op<SDNode atomic_op, bit IsInt = 1> {
multiclass noret_binary_atomic_op<SDNode atomic_op> {
let HasNoUse = true in
defm "_noret" : binary_atomic_op<atomic_op, IsInt>;
defm "_noret" : binary_atomic_op<atomic_op>;
}

multiclass noret_binary_atomic_op_fp<SDNode atomic_op> {
let HasNoUse = true in
defm "_noret" : binary_atomic_op_fp<atomic_op>;
}

multiclass noret_ternary_atomic_op<SDNode atomic_op> {
let HasNoUse = true in
defm "_noret" : ternary_atomic_op<atomic_op>;
}

multiclass binary_atomic_op_all_as<SDNode atomic_op, bit IsInt = 1> {
foreach as = [ "global", "flat", "constant", "local", "private", "region" ] in {
defvar atomic_addrspace_names = [ "global", "flat", "constant", "local", "private", "region" ];

multiclass binary_atomic_op_all_as<SDNode atomic_op> {
foreach as = atomic_addrspace_names in {
let AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {
defm "_"#as : binary_atomic_op<atomic_op>;
defm "_"#as : noret_binary_atomic_op<atomic_op>;
}
}
}
multiclass binary_atomic_op_fp_all_as<SDNode atomic_op> {
foreach as = atomic_addrspace_names in {
let AddressSpaces = !cast<AddressSpaceList>("LoadAddress_"#as).AddrSpaces in {
defm "_"#as : binary_atomic_op<atomic_op, IsInt>;
defm "_"#as : noret_binary_atomic_op<atomic_op, IsInt>;
defm "_"#as : binary_atomic_op_fp<atomic_op>;
defm "_"#as : noret_binary_atomic_op_fp<atomic_op>;
}
}
}
Expand All @@ -666,11 +681,9 @@ defm atomic_load_sub : binary_atomic_op_all_as<atomic_load_sub>;
defm atomic_load_umax : binary_atomic_op_all_as<atomic_load_umax>;
defm atomic_load_umin : binary_atomic_op_all_as<atomic_load_umin>;
defm atomic_load_xor : binary_atomic_op_all_as<atomic_load_xor>;
defm atomic_load_fadd : binary_atomic_op_all_as<atomic_load_fadd, 0>;
defm atomic_load_fadd : binary_atomic_op_fp_all_as<atomic_load_fadd>;
defm atomic_load_uinc_wrap : binary_atomic_op_all_as<atomic_load_uinc_wrap>;
defm atomic_load_udec_wrap : binary_atomic_op_all_as<atomic_load_udec_wrap>;
let MemoryVT = v2f16 in
defm atomic_load_fadd_v2f16 : binary_atomic_op_all_as<atomic_load_fadd, 0>;
defm AMDGPUatomic_cmp_swap : binary_atomic_op_all_as<AMDGPUatomic_cmp_swap>;

def load_align8_local : PatFrag<(ops node:$ptr), (load_local node:$ptr)>,
Expand Down
4 changes: 2 additions & 2 deletions llvm/lib/Target/AMDGPU/BUFInstructions.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1558,7 +1558,7 @@ multiclass BufferAtomicPat_Common<string OpPrefix, ValueType vt, string Inst, bi

defvar Op = !cast<SDPatternOperator>(OpPrefix
# !if(!eq(RtnMode, "ret"), "", "_noret")
# !if(isIntr, "", "_" # vt.Size));
# !if(isIntr, "", "_" # vt));
defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", "");

let AddedComplexity = !if(!eq(RtnMode, "ret"), 0, 1) in {
Expand Down Expand Up @@ -1595,7 +1595,7 @@ multiclass BufferAtomicCmpSwapPat_Common<ValueType vt, ValueType data_vt, string

defvar Op = !cast<SDPatternOperator>("AMDGPUatomic_cmp_swap_global"
# !if(!eq(RtnMode, "ret"), "", "_noret")
# "_" # vt.Size);
# "_" # vt);
defvar InstSuffix = !if(!eq(RtnMode, "ret"), "_RTN", "");
defvar data_vt_RC = getVregSrcForVT<data_vt>.ret.RegClass;

Expand Down
Loading

0 comments on commit af517f9

Please sign in to comment.